From 593eae7da4bd16468d84e47aeb2ba1bfef28b6a8 Mon Sep 17 00:00:00 2001 From: Lorenz Stechauner Date: Wed, 11 Oct 2023 23:43:45 +0200 Subject: [PATCH] database: use bins instead of buckets --- sql/v01/10.create.sql | 51 +++++++++++++++++++++-------------- sql/v01/20.view.sql | 10 +++---- sql/v01/99.schema_version.sql | 2 +- wgmaster/import.py | 2 +- wgmaster/migrate.py | 31 ++++++++++----------- 5 files changed, 54 insertions(+), 42 deletions(-) diff --git a/sql/v01/10.create.sql b/sql/v01/10.create.sql index 947b834..2fde901 100644 --- a/sql/v01/10.create.sql +++ b/sql/v01/10.create.sql @@ -444,6 +444,16 @@ CREATE TABLE season ( start_date TEXT CHECK (start_date REGEXP '^[1-9][0-9]{3}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$'), end_date TEXT CHECK (end_date REGEXP '^[1-9][0-9]{3}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$'), + bin_1_name TEXT DEFAULT NULL, + bin_2_name TEXT DEFAULT NULL, + bin_3_name TEXT DEFAULT NULL, + bin_4_name TEXT DEFAULT NULL, + bin_5_name TEXT DEFAULT NULL, + bin_6_name TEXT DEFAULT NULL, + bin_7_name TEXT DEFAULT NULL, + bin_8_name TEXT DEFAULT NULL, + bin_9_name TEXT DEFAULT NULL, + CONSTRAINT pk_season PRIMARY KEY (year), CONSTRAINT fk_season_currency FOREIGN KEY (currency) REFERENCES currency (code) ON UPDATE CASCADE @@ -623,6 +633,27 @@ CREATE TABLE delivery_part_modifier ( ON DELETE RESTRICT ) STRICT; +CREATE TABLE delivery_part_bin ( + year INTEGER NOT NULL, + did INTEGER NOT NULL, + dpnr INTEGER NOT NULL, + + bin_1 INTEGER DEFAULT NULL, + bin_2 INTEGER DEFAULT NULL, + bin_3 INTEGER DEFAULT NULL, + bin_4 INTEGER DEFAULT NULL, + bin_5 INTEGER DEFAULT NULL, + bin_6 INTEGER DEFAULT NULL, + bin_7 INTEGER DEFAULT NULL, + bin_8 INTEGER DEFAULT NULL, + bin_9 INTEGER DEFAULT NULL, + + CONSTRAINT pk_delivery_part_bin PRIMARY KEY (year, did, dpnr), + CONSTRAINT fk_delivery_part_bin_delivery_part FOREIGN KEY (year, did, dpnr) REFERENCES delivery_part (year, did, dpnr) + ON UPDATE CASCADE + ON DELETE CASCADE +) STRICT; + CREATE TABLE payment_variant ( year INTEGER NOT NULL, avnr INTEGER NOT NULL, @@ -633,16 +664,6 @@ CREATE TABLE payment_variant ( test_variant INTEGER NOT NULL CHECK (test_variant IN (TRUE, FALSE)), calc_time INTEGER, - bucket_1_name TEXT DEFAULT NULL, - bucket_2_name TEXT DEFAULT NULL, - bucket_3_name TEXT DEFAULT NULL, - bucket_4_name TEXT DEFAULT NULL, - bucket_5_name TEXT DEFAULT NULL, - bucket_6_name TEXT DEFAULT NULL, - bucket_7_name TEXT DEFAULT NULL, - bucket_8_name TEXT DEFAULT NULL, - bucket_9_name TEXT DEFAULT NULL, - comment TEXT DEFAULT NULL, data TEXT NOT NULL, ctime INTEGER NOT NULL DEFAULT (UNIXEPOCH()), @@ -661,16 +682,6 @@ CREATE TABLE payment_delivery_part ( mod_abs INTEGER NOT NULL DEFAULT 0, mod_rel REAL NOT NULL DEFAULT 0, - bucket_1 INTEGER DEFAULT NULL, - bucket_2 INTEGER DEFAULT NULL, - bucket_3 INTEGER DEFAULT NULL, - bucket_4 INTEGER DEFAULT NULL, - bucket_5 INTEGER DEFAULT NULL, - bucket_6 INTEGER DEFAULT NULL, - bucket_7 INTEGER DEFAULT NULL, - bucket_8 INTEGER DEFAULT NULL, - bucket_9 INTEGER DEFAULT NULL, - price_1 INTEGER DEFAULT NULL, price_2 INTEGER DEFAULT NULL, price_3 INTEGER DEFAULT NULL, diff --git a/sql/v01/20.view.sql b/sql/v01/20.view.sql index 53d79aa..e3ed042 100644 --- a/sql/v01/20.view.sql +++ b/sql/v01/20.view.sql @@ -23,7 +23,7 @@ SELECT p.year, p.did, p.dpnr, d.date, d.time, d.zwstid, d.lnr, d.lsnr, m.mgnr, m.family_name, m.given_name, p.sortid, p.weight, p.kmw, ROUND(p.kmw * (4.54 + 0.022 * p.kmw), 0) AS oe, p.qualid, p.hkid, p.kgnr, p.rdnr, - p.qualid NOT IN (SELECT l.qualid FROM wine_quality_level l WHERE p.kmw >= l.min_kmw OR l.min_kmw IS NULL) AS abgewertet, + p.qualid IN (SELECT l.qualid FROM wine_quality_level l WHERE NOT l.predicate AND (p.kmw >= l.min_kmw OR l.min_kmw IS NULL) ORDER BY l.min_kmw DESC LIMIT 1,100) AS abgewertet, p.qualid NOT IN ('WEI', 'RSW', 'LDW') AS min_quw, GROUP_CONCAT(DISTINCT a.attrid) as attributes, GROUP_CONCAT(DISTINCT o.modid) as modifiers, d.comment, p.comment as part_comment @@ -35,13 +35,13 @@ FROM delivery_part p GROUP BY p.year, p.did, p.dpnr ORDER BY p.year, p.did, p.dpnr; -CREATE VIEW v_bucket AS +CREATE VIEW v_bin AS SELECT year, mgnr, - sortid || IIF(min_quw, REPLACE(COALESCE(attributes, ''), ',', ''), '_') AS bucket, + sortid || IIF(min_quw, REPLACE(COALESCE(attributes, ''), ',', ''), '_') AS bin, SUM(weight) AS weight FROM v_delivery -GROUP BY year, mgnr, bucket -ORDER BY year, mgnr, LENGTH(bucket) DESC, bucket; +GROUP BY year, mgnr, bin +ORDER BY year, mgnr, LENGTH(bin) DESC, bin; CREATE VIEW v_stat_season AS SELECT year, diff --git a/sql/v01/99.schema_version.sql b/sql/v01/99.schema_version.sql index 0063e3a..c8f6b40 100644 --- a/sql/v01/99.schema_version.sql +++ b/sql/v01/99.schema_version.sql @@ -1,3 +1,3 @@ -- This value MUST NOT be changed while other connections are open! -PRAGMA schema_version = 200; +PRAGMA schema_version = 300; diff --git a/wgmaster/import.py b/wgmaster/import.py index a860173..a02639a 100755 --- a/wgmaster/import.py +++ b/wgmaster/import.py @@ -15,7 +15,7 @@ DIR: str TABLES = ['client_parameter', 'branch', 'wb_gl', 'wb_kg', 'wb_rd', 'wine_attribute', 'wine_cultivation', 'area_commitment_type', 'member', 'member_billing_address', 'member_telephone_number', 'member_email_address', 'area_commitment', 'season', 'modifier', 'delivery', 'delivery_part', 'delivery_part_attribute', 'delivery_part_modifier', - 'payment_variant', 'payment_delivery_part'] + 'delivery_part_bin', 'payment_variant', 'payment_delivery_part'] def get_sql_files() -> List[str]: diff --git a/wgmaster/migrate.py b/wgmaster/migrate.py index d2a86e6..b6cea5f 100755 --- a/wgmaster/migrate.py +++ b/wgmaster/migrate.py @@ -1510,10 +1510,11 @@ def migrate_deliveries(in_dir: str, out_dir: str) -> None: with utils.csv_open(f'{out_dir}/season.csv') as f_season, \ utils.csv_open(f'{out_dir}/modifier.csv') as f_mod: - f_season.header('year', 'currency', 'precision', 'start_date', 'end_date') + f_season.header('year', 'currency', 'precision', 'start_date', 'end_date', + 'bin_1_name', 'bin_2_name') f_mod.header('year', 'modid', 'ordering', 'name', 'abs', 'rel', 'standard', 'quick_select') for y, s in seasons.items(): - f_season.row(y, s['currency'], s['precision'], s['start'], s['end']) + f_season.row(y, s['currency'], s['precision'], s['start'], s['end'], 'gebunden', 'ungebunden') for m in modifiers.values(): abs_v = round(m['AZAS'] * pow(10, s['precision'])) if m['AZAS'] is not None else None rel_v = m['AZASProzent'] / 100.0 if m['AZASProzent'] is not None else None @@ -1533,8 +1534,7 @@ def migrate_payments(in_dir: str, out_dir: str) -> None: qual_map = {i: [s for s in p_qual if s['AZNR'] == i] for i in set([s['AZNR'] for s in p_qual])} with utils.csv_open(f'{out_dir}/payment_variant.csv') as f_payment: - f_payment.header('year', 'avnr', 'name', 'date', 'test_variant', 'calc_time', - 'bucket_1_name', 'bucket_2_name', 'bucket_3_name', 'comment', 'data') + f_payment.header('year', 'avnr', 'name', 'date', 'test_variant', 'calc_time', 'comment', 'data') for p in utils.csv_parse_dict(f'{in_dir}/TAuszahlung.csv'): year = p['Lesejahr'] if year is None: @@ -1633,7 +1633,7 @@ def migrate_payments(in_dir: str, out_dir: str) -> None: variant_year_map[year] = [] variant_year_map[year].append((p['AZNR'], year_map[year], p['TeilzahlungNr'])) f_payment.row(year, year_map[year], p['Titel'], p['Datum'], test, None, - 'Gebunden', 'Nicht gebunden', 'Abgewertet', p['Beschreibung'], json.dumps(data)) + p['Beschreibung'], json.dumps(data)) def get_modifiers(modifiers: str) -> Tuple[int, float]: if modifiers is None or modifiers == '': @@ -1665,27 +1665,28 @@ def migrate_payments(in_dir: str, out_dir: str) -> None: prec = pow(10, WGMASTER_PRECISION) return round(p1 * prec), round(p2 * prec), round(p3 * prec) - with utils.csv_open(f'{out_dir}/payment_delivery_part.csv') as f_del_pay: - f_del_pay.header('year', 'did', 'dpnr', 'avnr', 'bucket_1', 'bucket_2', 'bucket_3', 'amount') + # TODO database migration + with utils.csv_open(f'{out_dir}/payment_delivery_part.csv') as f_del_pay, \ + utils.csv_open(f'{out_dir}/delivery_part_bin.csv') as f_bin: + f_del_pay.header('year', 'did', 'dpnr', 'avnr', 'amount') + f_bin.header('year', 'did', 'dpnr', 'bin_1', 'bin_2') deliveries = {d['LINR']: d for d in utils.csv_parse_dict(f'{in_dir}/TLieferungen.csv')} for linr, (y, did, dpnr) in DELIVERY_MAP.items(): p = deliveries[linr] if y not in variant_year_map: continue + gew, geb_gew = int(p['Gewicht']), int(p['BGewichtGebunden']) + b2 = gew - geb_gew + b1 = geb_gew + f_bin.row(y, did, dpnr, b1, b2) for aznr, avnr, tznr in variant_year_map[y]: val = p[f'BTeilzahlung{tznr}' if tznr < 6 else 'BEndauszahlung'] val = round(val * pow(10, WGMASTER_PRECISION)) - b1, b2, b3 = 0, 0, 0 # prices = get_prices(aznr, p['SNR'], p['SANR'], int(p['Oechsle'])) # mod = get_modifiers(p['BAbschlaegeString']) # if not az_map[aznr].get('AbschlägeBerücksichtigen', False): # mod = 0, 0.0 - gew, geb_gew = int(p['Gewicht']), int(p['BGewichtGebunden']) - if QUAL_MAP[p['QSNR']] == 'WEI': - b3 += gew - else: - b2 += gew - geb_gew - b1 += geb_gew + # check_val = b1 * (prices[0] + mod[0]) + b2 * (prices[1] + mod[0]) + b3 * (prices[2] + mod[0]) # check_val *= 1 + mod[1] # check_val = round(check_val / 100) * 100 @@ -1693,7 +1694,7 @@ def migrate_payments(in_dir: str, out_dir: str) -> None: # print(p['LINR'], y, did, dpnr, avnr, val, check_val) # else: # print("GOOD") - f_del_pay.row(y, did, dpnr, avnr, b1, b2, b3, val) + f_del_pay.row(y, did, dpnr, avnr, val) def migrate_parameters(in_dir: str, out_dir: str) -> None: