neues skript sql_dump_diff.py zum Vergleich von 2 sql-dumps.
- leere, frisch initialisierte Datenbanken der Versionen verglichen - datasetprogramme erkennen jetzt auch suche nach standardtabellen wie uom
This commit is contained in:
@@ -0,0 +1,268 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
sql_dump_diff.py
|
||||
|
||||
Vergleicht zwei mit `pg_dump` (Klartext-Format, --format=plain) erzeugte
|
||||
SQL-Dumps strukturell: welche Tabellen/Spalten gibt es in Version A, die
|
||||
es in B nicht (mehr) gibt (und umgekehrt), welche Spaltentypen haben sich
|
||||
geändert, und wie viele Zeilen stehen jeweils in einer gemeinsamen
|
||||
Tabelle. Gedacht für zwei frisch initialisierte (oder später: mit
|
||||
denselben Proteus-Testdaten befüllte) Datenbanken zweier Tryton-Versionen
|
||||
- als dritte, von der Code-Analyse (extract_updates.py/model_diff.py)
|
||||
unabhängige Quelle für "was hat sich am Schema wirklich geändert".
|
||||
|
||||
Mit --model-diff (Ausgabe von model_diff.py) wird zusätzlich pro
|
||||
betroffener Tabelle geprüft, ob die rein statische Code-Analyse dieselbe
|
||||
Änderung schon vorhergesagt hatte (Modellname -> Tabellenname: Punkte
|
||||
durch Unterstriche ersetzen, wie es extract_updates.py/
|
||||
compare_snapshots.py auch tun) - zeigt, wie gut beide Ansätze
|
||||
übereinstimmen, und macht Abweichungen sichtbar (z. B. Python-seitige
|
||||
Änderungen ohne SQL-Schemaauswirkung, oder Tabellenänderungen durch ein
|
||||
Modul, das nicht im --modules-Set von extract_model_tree.py steckte).
|
||||
|
||||
Zeilen werden bereits vollständig geparst (als dict pro Zeile, Spalte ->
|
||||
Wert) und im Rückgabewert von parse_dump() mitgeführt - für eine spätere
|
||||
inhaltliche Vergleichs-Erweiterung (z. B. per --key-column concrete Werte
|
||||
statt nur Zeilenzahlen gegenüberstellen). Der aktuelle Report nutzt davon
|
||||
bisher nur die Zeilenzahl.
|
||||
|
||||
Bekannte Grenzen: erkennt keine Indizes/Constraints/Sequences/
|
||||
Spaltenreihenfolge, nur Tabellen + Spalten(typen) + Zeilenzahl.
|
||||
|
||||
Nutzung:
|
||||
python3 sql_dump_diff.py --dump-a data/leer5.sql --label-a 5.0 \\
|
||||
--dump-b data/leer8.sql --label-b 8.0 \\
|
||||
--model-diff reports/model_diff.json \\
|
||||
--output reports/dump_diff_5.0_8.0.json \\
|
||||
--report reports/dump_diff_5.0_8.0.md
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
CREATE_TABLE_RE = re.compile(r'^CREATE TABLE public\.(\w+) \($')
|
||||
COPY_RE = re.compile(r'^COPY public\.(\w+) \(([^)]*)\) FROM stdin;$')
|
||||
|
||||
|
||||
def unescape_copy_value(raw):
|
||||
"""pg_dump-COPY-Escaping rückgängig machen (\\N -> NULL, \\t/\\n/\\\\ -> Zeichen)."""
|
||||
if raw == r'\N':
|
||||
return None
|
||||
out = []
|
||||
i = 0
|
||||
while i < len(raw):
|
||||
c = raw[i]
|
||||
if c == '\\' and i + 1 < len(raw):
|
||||
out.append({'t': '\t', 'n': '\n', 'r': '\r', '\\': '\\'}.get(raw[i + 1], raw[i + 1]))
|
||||
i += 2
|
||||
else:
|
||||
out.append(c)
|
||||
i += 1
|
||||
return ''.join(out)
|
||||
|
||||
|
||||
def parse_column_line(line):
|
||||
"""Eine Zeile aus dem CREATE-TABLE-Block -> (spaltenname, typ) oder (None, None)."""
|
||||
text = line.strip()
|
||||
if not text or text.startswith(('CONSTRAINT', 'PRIMARY KEY', 'UNIQUE')):
|
||||
return None, None
|
||||
text = text.rstrip(',')
|
||||
for stopper in (' NOT NULL', ' DEFAULT ', ' COLLATE '):
|
||||
idx = text.find(stopper)
|
||||
if idx != -1:
|
||||
text = text[:idx]
|
||||
parts = text.split(None, 1)
|
||||
if len(parts) != 2:
|
||||
return None, None
|
||||
name, coltype = parts
|
||||
return name.strip('"'), coltype.strip()
|
||||
|
||||
|
||||
def parse_dump(path):
|
||||
"""Liefert {tabelle: {"columns": {name: typ}, "row_count": int, "rows": [dict, ...]}}."""
|
||||
tables = {}
|
||||
with open(path, encoding='utf-8', errors='replace') as f:
|
||||
lines = iter(f)
|
||||
for line in lines:
|
||||
stripped = line.rstrip('\n')
|
||||
|
||||
m = CREATE_TABLE_RE.match(stripped)
|
||||
if m:
|
||||
table = m.group(1)
|
||||
columns = {}
|
||||
for col_line in lines:
|
||||
col_line = col_line.rstrip('\n')
|
||||
if col_line.startswith(');'):
|
||||
break
|
||||
name, coltype = parse_column_line(col_line)
|
||||
if name:
|
||||
columns[name] = coltype
|
||||
entry = tables.setdefault(table, {"columns": {}, "row_count": 0, "rows": []})
|
||||
entry["columns"] = columns
|
||||
continue
|
||||
|
||||
m = COPY_RE.match(stripped)
|
||||
if m:
|
||||
table, col_csv = m.group(1), m.group(2)
|
||||
col_names = [c.strip() for c in col_csv.split(',')]
|
||||
rows = []
|
||||
for data_line in lines:
|
||||
data_line = data_line.rstrip('\n')
|
||||
if data_line == '\\.':
|
||||
break
|
||||
values = [unescape_copy_value(v) for v in data_line.split('\t')]
|
||||
rows.append(dict(zip(col_names, values)))
|
||||
entry = tables.setdefault(table, {"columns": {}, "row_count": 0, "rows": []})
|
||||
entry["row_count"] = len(rows)
|
||||
entry["rows"] = rows
|
||||
|
||||
return tables
|
||||
|
||||
|
||||
def diff_dumps(tables_a, tables_b):
|
||||
tables_removed = sorted(set(tables_a) - set(tables_b))
|
||||
tables_added = sorted(set(tables_b) - set(tables_a))
|
||||
common = sorted(set(tables_a) & set(tables_b))
|
||||
|
||||
changed = {}
|
||||
unchanged = []
|
||||
for table in common:
|
||||
cols_a, cols_b = tables_a[table]["columns"], tables_b[table]["columns"]
|
||||
added_cols = sorted(set(cols_b) - set(cols_a))
|
||||
removed_cols = sorted(set(cols_a) - set(cols_b))
|
||||
type_changes = {
|
||||
c: [cols_a[c], cols_b[c]]
|
||||
for c in sorted(set(cols_a) & set(cols_b))
|
||||
if cols_a[c] != cols_b[c]
|
||||
}
|
||||
row_count_a, row_count_b = tables_a[table]["row_count"], tables_b[table]["row_count"]
|
||||
|
||||
schema_changed = bool(added_cols or removed_cols or type_changes)
|
||||
if schema_changed or row_count_a != row_count_b:
|
||||
if removed_cols or type_changes:
|
||||
expected_category = "geändert"
|
||||
elif added_cols:
|
||||
expected_category = "nur ergänzt"
|
||||
else:
|
||||
expected_category = None # nur Zeilenzahl anders - keine Code-Definitionsänderung
|
||||
changed[table] = {
|
||||
"added_columns": added_cols,
|
||||
"removed_columns": removed_cols,
|
||||
"type_changes": type_changes,
|
||||
"row_count_a": row_count_a,
|
||||
"row_count_b": row_count_b,
|
||||
"schema_changed": schema_changed,
|
||||
"expected_category": expected_category,
|
||||
}
|
||||
else:
|
||||
unchanged.append(table)
|
||||
|
||||
return {
|
||||
"tables_removed": tables_removed,
|
||||
"tables_added": tables_added,
|
||||
"tables_changed": changed,
|
||||
"tables_unchanged": unchanged,
|
||||
}
|
||||
|
||||
|
||||
def build_table_to_models(model_diff):
|
||||
mapping = {}
|
||||
for model_name in model_diff:
|
||||
mapping.setdefault(model_name.replace(".", "_"), []).append(model_name)
|
||||
return mapping
|
||||
|
||||
|
||||
def model_diff_note(table, model_diff, table_to_models, expected_category):
|
||||
"""Vergleicht einen Tabellen-Fund mit der Vorhersage aus model_diff.py.
|
||||
Liefert einen kurzen Text: Übereinstimmung, Abweichung, oder kein
|
||||
passendes Modell gefunden (z. B. reine Relations-/Interntabelle)."""
|
||||
if not model_diff:
|
||||
return ""
|
||||
models = table_to_models.get(table)
|
||||
if not models:
|
||||
return " _(kein passendes Modell in model_diff gefunden)_"
|
||||
parts = []
|
||||
for model in models:
|
||||
category = model_diff[model]["category"]
|
||||
flag = "übereinstimmung" if category == expected_category else "ABWEICHUNG"
|
||||
parts.append(f"`{model}`={category} [{flag}]")
|
||||
return " _(" + "; ".join(parts) + ")_"
|
||||
|
||||
|
||||
def write_report(diff, out_path, label_a, label_b, model_diff, table_to_models):
|
||||
lines = [f"# SQL-Dump-Diff: {label_a} -> {label_b}\n"]
|
||||
|
||||
lines.append(f"\n## Tabellen entfernt ({len(diff['tables_removed'])})\n")
|
||||
for t in diff["tables_removed"]:
|
||||
lines.append(f"- `{t}`{model_diff_note(t, model_diff, table_to_models, 'entfernt')}")
|
||||
|
||||
lines.append(f"\n\n## Tabellen neu ({len(diff['tables_added'])})\n")
|
||||
for t in diff["tables_added"]:
|
||||
lines.append(f"- `{t}`{model_diff_note(t, model_diff, table_to_models, 'neu')}")
|
||||
|
||||
schema_changed = {t: i for t, i in diff["tables_changed"].items() if i["schema_changed"]}
|
||||
rows_only = {t: i for t, i in diff["tables_changed"].items() if not i["schema_changed"]}
|
||||
|
||||
lines.append(f"\n\n## Tabellen mit Schema-Änderung ({len(schema_changed)})\n")
|
||||
lines.append(
|
||||
"Spalten wurden hinzugefügt/entfernt oder haben einen anderen Typ - "
|
||||
"vergleichbar mit einer model_diff.py-Kategorie.\n"
|
||||
)
|
||||
for t, info in sorted(schema_changed.items()):
|
||||
note = model_diff_note(t, model_diff, table_to_models, info["expected_category"])
|
||||
lines.append(f"\n### `{t}`{note}")
|
||||
if info["added_columns"]:
|
||||
lines.append(f"- neue Spalten: {', '.join(info['added_columns'])}")
|
||||
if info["removed_columns"]:
|
||||
lines.append(f"- entfernte Spalten: {', '.join(info['removed_columns'])}")
|
||||
for col, (ta, tb) in info["type_changes"].items():
|
||||
lines.append(f"- **{col}**: {ta} -> {tb}")
|
||||
if info["row_count_a"] != info["row_count_b"]:
|
||||
lines.append(f"- Zeilenzahl: {info['row_count_a']} -> {info['row_count_b']}")
|
||||
|
||||
lines.append(f"\n\n## Nur Zeilenzahl unterschiedlich, Schema identisch ({len(rows_only)})\n")
|
||||
lines.append(
|
||||
"Keine Code-Definitionsänderung zu erwarten (z. B. mehr XML-Fixture-Daten in "
|
||||
"der neueren Version) - daher kein model_diff-Abgleich.\n"
|
||||
)
|
||||
for t, info in sorted(rows_only.items()):
|
||||
lines.append(f"- `{t}`: {info['row_count_a']} -> {info['row_count_b']}")
|
||||
|
||||
lines.append(f"\n\n## Unverändert ({len(diff['tables_unchanged'])})\n")
|
||||
lines.append(", ".join(f"`{t}`" for t in diff["tables_unchanged"]))
|
||||
|
||||
out_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--dump-a", required=True)
|
||||
parser.add_argument("--dump-b", required=True)
|
||||
parser.add_argument("--label-a", default="A")
|
||||
parser.add_argument("--label-b", default="B")
|
||||
parser.add_argument("--model-diff", default=None, help="Optional: model_diff.json zum Abgleich")
|
||||
parser.add_argument("--output", default="reports/dump_diff.json")
|
||||
parser.add_argument("--report", default="reports/dump_diff.md")
|
||||
args = parser.parse_args()
|
||||
|
||||
tables_a = parse_dump(args.dump_a)
|
||||
tables_b = parse_dump(args.dump_b)
|
||||
diff = diff_dumps(tables_a, tables_b)
|
||||
|
||||
model_diff = json.loads(Path(args.model_diff).read_text(encoding="utf-8")) if args.model_diff else {}
|
||||
table_to_models = build_table_to_models(model_diff) if model_diff else {}
|
||||
|
||||
Path(args.output).write_text(json.dumps(diff, indent=2, ensure_ascii=False, sort_keys=True), encoding="utf-8")
|
||||
write_report(diff, Path(args.report), args.label_a, args.label_b, model_diff, table_to_models)
|
||||
|
||||
print(
|
||||
f"{len(diff['tables_removed'])} entfernt, {len(diff['tables_added'])} neu, "
|
||||
f"{len(diff['tables_changed'])} geändert, {len(diff['tables_unchanged'])} unverändert "
|
||||
f"-> {args.output}, {args.report}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user