From d5c8c224df1edda2737203bf7b9339a2f5b1455c Mon Sep 17 00:00:00 2001 From: Lorenz Stechauner Date: Fri, 14 Apr 2023 20:39:04 +0200 Subject: [PATCH] Update database schema --- sql/v01/10.create.sql | 59 ++++++++++++++++++++++++++++-------------- sql/v01/20.view.sql | 59 ++++++++++++++++++++++++++++++++++++++++++ wgmaster/import.py | 4 +-- wgmaster/migrate.py | 60 +++++++++++++++++++++++++++++-------------- 4 files changed, 142 insertions(+), 40 deletions(-) diff --git a/sql/v01/10.create.sql b/sql/v01/10.create.sql index 6f35041..c6ff80f 100644 --- a/sql/v01/10.create.sql +++ b/sql/v01/10.create.sql @@ -359,7 +359,6 @@ CREATE TABLE area_commitment ( area INTEGER NOT NULL, sortid TEXT NOT NULL, - attrid TEXT, cultid TEXT NOT NULL, CONSTRAINT pk_area_commitment PRIMARY KEY (vnr, kgnr, gstnr), @@ -369,9 +368,6 @@ CREATE TABLE area_commitment ( CONSTRAINT fk_area_commitment_wine_variety FOREIGN KEY (sortid) REFERENCES wine_variety (sortid) ON UPDATE CASCADE ON DELETE RESTRICT, - CONSTRAINT fk_area_commitment_wine_attribute FOREIGN KEY (attrid) REFERENCES wine_attribute (attrid) - ON UPDATE CASCADE - ON DELETE RESTRICT, CONSTRAINT fk_area_commitment_wine_cultivation FOREIGN KEY (cultid) REFERENCES wine_cultivation (cultid) ON UPDATE CASCADE ON DELETE RESTRICT, @@ -380,6 +376,21 @@ CREATE TABLE area_commitment ( ON DELETE RESTRICT ) STRICT; +CREATE TABLE area_commitment_attribute ( + vnr INTEGER NOT NULL, + kgnr INTEGER NOT NULL, + gstnr TEXT NOT NULL, + attrid TEXT NOT NULL, + + CONSTRAINT pk_area_commitment_attribute PRIMARY KEY (vnr, kgnr, gstnr, attrid), + CONSTRAINT fk_area_commitment_attribute_area_commitment FOREIGN KEY (vnr, kgnr, gstnr) REFERENCES area_commitment (vnr, kgnr, gstnr) + ON UPDATE CASCADE + ON DELETE CASCADE, + CONSTRAINT fk_area_commitment_attribute_wine_attribute FOREIGN KEY (attrid) REFERENCES wine_attribute (attrid) + ON UPDATE CASCADE + ON DELETE RESTRICT +) STRICT; + ---------------------------------------------------------------- @@ -400,7 +411,7 @@ CREATE TABLE season ( CREATE TABLE modifier ( year INTEGER NOT NULL, - mnr INTEGER NOT NULL, + modid TEXT NOT NULL CHECK (modid REGEXP '^[A-Z]+$'), name TEXT NOT NULL, abs INTEGER, @@ -409,7 +420,7 @@ CREATE TABLE modifier ( standard INTEGER NOT NULL CHECK (standard IN (TRUE, FALSE)), quick_select INTEGER NOT NULL CHECK (quick_select IN (TRUE, FALSE)), - CONSTRAINT pk_modifier PRIMARY KEY (year, mnr), + CONSTRAINT pk_modifier PRIMARY KEY (year, modid), CONSTRAINT fk_modifier_season FOREIGN KEY (year) REFERENCES season (year) ON UPDATE CASCADE ON DELETE CASCADE, @@ -459,8 +470,6 @@ CREATE TABLE delivery_part ( dpnr INTEGER NOT NULL, sortid TEXT NOT NULL, - attrid TEXT DEFAULT NULL, - weight INTEGER NOT NULL, kmw REAL NOT NULL, qualid TEXT NOT NULL, @@ -474,7 +483,7 @@ CREATE TABLE delivery_part ( spl_check INTEGER NOT NULL CHECK (spl_check IN (TRUE, FALSE)) DEFAULT FALSE, hand_picked INTEGER CHECK (hand_picked IN (TRUE, FALSE)) DEFAULT NULL, - lesemaschine INTEGER CHECK (lesemaschine IN (True, FALSE)) DEFAULT NULL, + lesewagen INTEGER CHECK (lesewagen IN (True, FALSE)) DEFAULT NULL, temperature REAL DEFAULT NULL, acid REAL DEFAULT NULL, @@ -491,9 +500,6 @@ CREATE TABLE delivery_part ( CONSTRAINT fk_delivery_part_wine_variety FOREIGN KEY (sortid) REFERENCES wine_variety (sortid) ON UPDATE CASCADE ON DELETE RESTRICT, - CONSTRAINT fk_delivery_part_wine_attribute FOREIGN KEY (attrid) REFERENCES wine_attribute (attrid) - ON UPDATE CASCADE - ON DELETE RESTRICT, CONSTRAINT fk_delivery_part_wine_quality FOREIGN KEY (qualid) REFERENCES wine_quality (qualid) ON UPDATE CASCADE ON DELETE RESTRICT, @@ -534,17 +540,32 @@ BEGIN ) WHERE (year, did, dpnr) = (NEW.year, NEW.did, NEW.dpnr); END; -CREATE TABLE delivery_part_modifier ( - year INTEGER NOT NULL, - did INTEGER NOT NULL, - dpnr INTEGER NOT NULL, - mnr INTEGER NOT NULL, +CREATE TABLE delivery_part_attribute ( + year INTEGER NOT NULL, + did INTEGER NOT NULL, + dpnr INTEGER NOT NULL, + attrid TEXT NOT NULL, - CONSTRAINT pk_delivery_part_modifier PRIMARY KEY (year, did, dpnr, mnr), + CONSTRAINT pk_delivery_part_attribute PRIMARY KEY (year, did, dpnr, attrid), + CONSTRAINT fk_delivery_part_attribute_delivery_part FOREIGN KEY (year, did, dpnr) REFERENCES delivery_part (year, did, dpnr) + ON UPDATE CASCADE + ON DELETE CASCADE, + CONSTRAINT fk_delivery_part_attribute_wine_attribute FOREIGN KEY (attrid) REFERENCES wine_attribute (attrid) + ON UPDATE CASCADE + ON DELETE RESTRICT +) STRICT; + +CREATE TABLE delivery_part_modifier ( + year INTEGER NOT NULL, + did INTEGER NOT NULL, + dpnr INTEGER NOT NULL, + modid TEXT NOT NULL, + + CONSTRAINT pk_delivery_part_modifier PRIMARY KEY (year, did, dpnr, modid), CONSTRAINT fk_delivery_part_modifier_delivery_part FOREIGN KEY (year, did, dpnr) REFERENCES delivery_part (year, did, dpnr) ON UPDATE CASCADE ON DELETE CASCADE, - CONSTRAINT fk_delivery_part_modifier_modifier FOREIGN KEY (year, mnr) REFERENCES modifier (year, mnr) + CONSTRAINT fk_delivery_part_modifier_modifier FOREIGN KEY (year, modid) REFERENCES modifier (year, modid) ON UPDATE CASCADE ON DELETE RESTRICT ) STRICT; diff --git a/sql/v01/20.view.sql b/sql/v01/20.view.sql index d5d555d..c97fb03 100644 --- a/sql/v01/20.view.sql +++ b/sql/v01/20.view.sql @@ -4,3 +4,62 @@ SELECT plz, p.dest AS bestimmungsort, g.name AS gemeinde, g.gkz, o.name AS ort, FROM AT_gem g JOIN AT_ort o ON o.gkz = g.gkz JOIN AT_plz_dest p ON p.okz = o.okz; + +CREATE VIEW v_delivery AS +SELECT p.year, p.did, p.dpnr, + d.date, d.time, d.zwstid, d.lnr, d.lsnr, + m.mgnr, m.family_name, m.given_name, + p.sortid, p.weight, p.kmw, ROUND(p.kmw * (4.54 + 0.022 * p.kmw), 0) AS oe, p.qualid, p.hkid, p.kgnr, + GROUP_CONCAT(DISTINCT a.attrid) as attributes, GROUP_CONCAT(DISTINCT o.modid) as modifiers, + d.comment, p.comment as part_comment +FROM delivery_part p + JOIN delivery d ON (d.year, d.did) = (p.year, p.did) + JOIN member m ON m.mgnr = d.mgnr + LEFT JOIN delivery_part_attribute a ON (a.year, a.did, a.dpnr) = (p.year, p.did, p.dpnr) + LEFT JOIN delivery_part_modifier o ON (o.year, o.did, o.dpnr) = (p.year, p.did, p.dpnr) +GROUP BY p.year, p.did, p.dpnr +ORDER BY p.year, p.did, p.dpnr; + +CREATE VIEW v_stat_season AS +SELECT year, + SUM(weight) AS sum, + ROUND(SUM(kmw * weight) / SUM(weight), 2) AS kmw, + ROUND(SUM(oe * weight) / SUM(weight), 1) AS oe, + COUNT(DISTINCT did) AS lieferungen, + COUNT(DISTINCT mgnr) AS mitglieder +FROM v_delivery +GROUP BY year +ORDER BY year; + +CREATE VIEW v_stat_sort AS +SELECT year, sortid, + SUM(weight) as sum, + ROUND(SUM(kmw * weight) / SUM(weight), 2) AS kmw, + ROUND(SUM(oe * weight) / SUM(weight), 1) AS oe, + COUNT(DISTINCT did) AS lieferungen, + COUNT(DISTINCT mgnr) AS mitglieder +FROM v_delivery +GROUP BY year, sortid +ORDER BY year, sortid; + +CREATE VIEW v_stat_attr AS +SELECT year, attributes, + SUM(weight) as sum, + ROUND(SUM(kmw * weight) / SUM(weight), 2) AS kmw, + ROUND(SUM(oe * weight) / SUM(weight), 1) AS oe, + COUNT(DISTINCT did) AS lieferungen, + COUNT(DISTINCT mgnr) AS mitglieder +FROM v_delivery +GROUP BY year, attributes +ORDER BY year, LENGTH(attributes) DESC, attributes; + +CREATE VIEW v_stat_sort_attr AS +SELECT year, sortid, attributes, + SUM(weight) as sum, + ROUND(SUM(kmw * weight) / SUM(weight), 2) AS kmw, + ROUND(SUM(oe * weight) / SUM(weight), 1) AS oe, + COUNT(DISTINCT did) AS lieferungen, + COUNT(DISTINCT mgnr) AS mitglieder +FROM v_delivery +GROUP BY year, sortid, attributes +ORDER BY year, sortid, LENGTH(attributes) DESC, attributes; diff --git a/wgmaster/import.py b/wgmaster/import.py index e40ac98..338508d 100755 --- a/wgmaster/import.py +++ b/wgmaster/import.py @@ -14,8 +14,8 @@ import csv DIR: str TABLES = ['branch', 'wb_gl', 'wb_kg', 'wb_rd', 'wine_attribute', 'wine_cultivation', - 'member', 'member_billing_address', 'contract', 'area_commitment', - 'season', 'modifier', 'delivery', 'delivery_part', 'delivery_part_modifier', ] + 'member', 'member_billing_address', 'contract', 'area_commitment', 'area_commitment_attribute', + 'season', 'modifier', 'delivery', 'delivery_part', 'delivery_part_attribute', 'delivery_part_modifier', ] # 'payment_variant', 'delivery_payment', 'member_payment'] diff --git a/wgmaster/migrate.py b/wgmaster/migrate.py index 1091c27..13c8c0c 100755 --- a/wgmaster/migrate.py +++ b/wgmaster/migrate.py @@ -331,6 +331,7 @@ def migrate_attributes(in_dir: str, out_dir: str) -> None: f.write(csv.format_row(a['SANR'], a['Attribut'], int(a['KgProHa']))) if WG == 'MATZEN': f.write(csv.format_row('M', 'Matzen', 10000)) + f.write(csv.format_row('HU', 'Huber', 10000)) def migrate_cultivations(in_dir: str, out_dir: str) -> None: @@ -649,9 +650,12 @@ def migrate_contracts(in_dir: str, out_dir: str) -> None: invalid(mgnr, 'GstNr.', f'{kgnr:05}-{nr_str}') return [] - with open(f'{out_dir}/contract.csv', 'w+') as f_c, open(f'{out_dir}/area_commitment.csv', 'w+') as f_fb: + with open(f'{out_dir}/contract.csv', 'w+') as f_c, \ + open(f'{out_dir}/area_commitment.csv', 'w+') as f_fb, \ + open(f'{out_dir}/area_commitment_attribute.csv', 'w+') as f_attr: f_c.write('vnr;mgnr;year_from;year_to\n') - f_fb.write('vnr;kgnr;gstnr;rdnr;area;sortid;attrid;cultid\n') + f_fb.write('vnr;kgnr;gstnr;rdnr;area;sortid;cultid\n') + f_attr.write('vnr;kgnr;gstnr;attrid\n') for fb in csv.parse_dict(f'{in_dir}/TFlaechenbindungen.csv'): if fb['Von'] is None and fb['Bis'] is None: @@ -682,8 +686,10 @@ def migrate_contracts(in_dir: str, out_dir: str) -> None: a = area - gst_area * (len(gstnrs) - 1) if i == 0 else gst_area rdnr = REED_MAP[fb['RNR']][1] if fb['RNR'] else None f_fb.write(csv.format_row( - vnr, kgnr, gstnr, rdnr, a, fb['SNR'], fb['SANR'], CULTIVATION_MAP[fb['BANR']] + vnr, kgnr, gstnr, rdnr, a, fb['SNR'], CULTIVATION_MAP[fb['BANR']] )) + if fb['SANR']: + f_attr.write(csv.format_row(vnr, kgnr, gstnr, fb['SANR'])) def fix_deliveries(deliveries: Iterable[Dict[str, Any]]) -> Iterable[Tuple[str, List[int], datetime.date]]: @@ -744,15 +750,24 @@ def migrate_deliveries(in_dir: str, out_dir: str) -> None: seasons = {} branches = {} + for mod in modifiers.values(): + name: str = mod['Bezeichnung'] + if WG == 'MATZEN': + mod['id'] = name[-1] if name.startswith('Klasse') else 'TB' if name == 'Treuebonus' else 'PZS' + else: + raise NotImplementedError() + deliveries = list(csv.parse_dict(f'{in_dir}/TLieferungen.csv')) delivery_dict = {d['LINR']: d for d in deliveries} fixed = fix_deliveries(deliveries) with open(f'{out_dir}/delivery.csv', 'w+') as f_delivery, \ - open(f'{out_dir}/delivery_part.csv', 'w+') as f_part: + open(f'{out_dir}/delivery_part.csv', 'w+') as f_part, \ + open(f'{out_dir}/delivery_part_attribute.csv', 'w+') as f_attr: f_delivery.write('year;did;date;time;zwstid;lnr;lsnr;mgnr;comment\n') - f_part.write('year;did;dpnr;sortid;attrid;weight;kmw;qualid;hkid;kgnr;rdnr;gerebelt;manual_weighing;spl_check;' - 'hand_picked;lesemaschine;temperature;acid;scale_id;weighing_id;comment\n') + f_part.write('year;did;dpnr;sortid;weight;kmw;qualid;hkid;kgnr;rdnr;gerebelt;manual_weighing;spl_check;' + 'hand_picked;lesewagen;temperature;acid;scale_id;weighing_id;comment\n') + f_attr.write('year;did;dpnr;attrid\n') for lsnr, linrs, date in fixed: if date.year not in seasons: @@ -778,6 +793,7 @@ def migrate_deliveries(in_dir: str, out_dir: str) -> None: lnr = branches[znr][date] comments = [] + attributes = set() for dpnr, linr in enumerate(linrs, start=1): d = delivery_dict[linr] delivery_map[linr] = (date.year, snr, dpnr) @@ -786,28 +802,32 @@ def migrate_deliveries(in_dir: str, out_dir: str) -> None: oe = d['OechsleOriginal'] or d['Oechsle'] kmw = GRADATION_MAP[oe] - sortid, attrid = d['SNR'].upper(), d['SANR'] or None + sortid = d['SNR'].upper() + if d['SANR']: + attributes.add(d['SANR']) if len(sortid) != 2: - attrid = sortid[-1] + attributes.add(sortid[2:]) sortid = sortid[:2] if WG == 'MATZEN': if sortid == 'HU': # Gr.Veltliner (Huber) sortid = 'GV' + attributes.add('HU') elif sortid == 'SV': # FIXME probably Sortenverschnitt? sortid = 'SW' elif sortid == 'WC': - # WEIßBURGUNDER/CHARDONNAY + # WEIẞBURGUNDER/CHARDONNAY sortid = 'SW' - if attrid == 'H': - attrid = 'HK' - elif attrid == 'W': - attrid = None + if 'H' in attributes: + attributes.remove('H') + attributes.add('HK') + if 'W' in attributes: + attributes.remove('W') if d['SNR'] != sortid: - print(f'{d["SNR"]}/{d["SANR"]} -> {sortid}/{attrid}') + print(f'{d["SNR"]}/{d["SANR"]} -> {sortid}/{attributes}') kgnr, rdnr = None, None if d['GNR']: @@ -851,32 +871,34 @@ def migrate_deliveries(in_dir: str, out_dir: str) -> None: comments.append(comment) f_part.write(csv.format_row( - date.year, snr, dpnr, sortid, attrid, int(d['Gewicht']), kmw, QUAL_MAP[d['QSNR']], HKID, kgnr, rdnr, + date.year, snr, dpnr, sortid, int(d['Gewicht']), kmw, QUAL_MAP[d['QSNR']], HKID, kgnr, rdnr, d['Gerebelt'] or False, d['Handwiegung'] or False, d['Spaetlese-Ueberpruefung'] or False, hand, lesemaschine, d['Temperatur'], acid, scale_id, weighing_id, comment )) + for attrid in attributes: + f_attr.write(csv.format_row(date.year, snr, dpnr, attrid)) f_delivery.write(csv.format_row( date.year, snr, date, d['Uhrzeit'], BRANCH_MAP[d['ZNR']], lnr, lsnr, d['MGNR'], '; '.join(comments) or None )) with open(f'{out_dir}/delivery_part_modifier.csv', 'w+') as f_part_mod: - f_part_mod.write('year;did;dpnr;mnr\n') + f_part_mod.write('year;did;dpnr;modid\n') for m in csv.parse_dict(f'{in_dir}/TLieferungAbschlag.csv'): if m['LINR'] not in delivery_map: continue nid = delivery_map[m['LINR']] - f_part_mod.write(csv.format_row(nid[0], nid[1], nid[2], m['ASNR'])) + f_part_mod.write(csv.format_row(nid[0], nid[1], nid[2], modifiers[m['ASNR']]['id'])) with open(f'{out_dir}/season.csv', 'w+') as f_season, open(f'{out_dir}/modifier.csv', 'w+') as f_mod: f_season.write('year;currency;precision;start_date;end_date\n') - f_mod.write('year;mnr;name;abs;rel;standard;quick_select\n') + f_mod.write('year;modid;name;abs;rel;standard;quick_select\n') for y, s in seasons.items(): f_season.write(csv.format_row(y, s['currency'], s['precision'], s['start'], s['end'])) for m in modifiers.values(): abs_v = int(m['AZAS'] * pow(10, s['precision'])) if m['AZAS'] is not None else None f_mod.write(csv.format_row( - y, m['ASNR'], m['Bezeichnung'], abs_v, m['AZASProzent'], m['Standard'], m['Schnellauswahl'] + y, m['id'], m['Bezeichnung'], abs_v, m['AZASProzent'], m['Standard'], m['Schnellauswahl'] ))