winzer/elwig-misc
3
1
Fork 0
Code Pull Requests Activity

database: use bins instead of buckets

Browse Source
This commit is contained in:
Lorenz Stechauner
2023-10-11 23:43:45 +02:00
parent a70276adf8
commit 593eae7da4
5 changed files with 54 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
sql/v01/10.create.sql
View File

@ -444,6 +444,16 @@ CREATE TABLE season (
start_date TEXT CHECK (start_date REGEXP '^[1-9][0-9]{3}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$'), start_date TEXT CHECK (start_date REGEXP '^[1-9][0-9]{3}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$'),
end_date TEXT CHECK (end_date REGEXP '^[1-9][0-9]{3}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$'), end_date TEXT CHECK (end_date REGEXP '^[1-9][0-9]{3}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$'),
bin_1_name TEXT DEFAULT NULL,
bin_2_name TEXT DEFAULT NULL,
bin_3_name TEXT DEFAULT NULL,
bin_4_name TEXT DEFAULT NULL,
bin_5_name TEXT DEFAULT NULL,
bin_6_name TEXT DEFAULT NULL,
bin_7_name TEXT DEFAULT NULL,
bin_8_name TEXT DEFAULT NULL,
bin_9_name TEXT DEFAULT NULL,
CONSTRAINT pk_season PRIMARY KEY (year), CONSTRAINT pk_season PRIMARY KEY (year),
CONSTRAINT fk_season_currency FOREIGN KEY (currency) REFERENCES currency (code) CONSTRAINT fk_season_currency FOREIGN KEY (currency) REFERENCES currency (code)
ON UPDATE CASCADE ON UPDATE CASCADE
@ -623,6 +633,27 @@ CREATE TABLE delivery_part_modifier (
ON DELETE RESTRICT ON DELETE RESTRICT
) STRICT; ) STRICT;
CREATE TABLE delivery_part_bin (
year INTEGER NOT NULL,
did INTEGER NOT NULL,
dpnr INTEGER NOT NULL,
bin_1 INTEGER DEFAULT NULL,
bin_2 INTEGER DEFAULT NULL,
bin_3 INTEGER DEFAULT NULL,
bin_4 INTEGER DEFAULT NULL,
bin_5 INTEGER DEFAULT NULL,
bin_6 INTEGER DEFAULT NULL,
bin_7 INTEGER DEFAULT NULL,
bin_8 INTEGER DEFAULT NULL,
bin_9 INTEGER DEFAULT NULL,
CONSTRAINT pk_delivery_part_bin PRIMARY KEY (year, did, dpnr),
CONSTRAINT fk_delivery_part_bin_delivery_part FOREIGN KEY (year, did, dpnr) REFERENCES delivery_part (year, did, dpnr)
ON UPDATE CASCADE
ON DELETE CASCADE
) STRICT;
CREATE TABLE payment_variant ( CREATE TABLE payment_variant (
year INTEGER NOT NULL, year INTEGER NOT NULL,
avnr INTEGER NOT NULL, avnr INTEGER NOT NULL,
@ -633,16 +664,6 @@ CREATE TABLE payment_variant (
test_variant INTEGER NOT NULL CHECK (test_variant IN (TRUE, FALSE)), test_variant INTEGER NOT NULL CHECK (test_variant IN (TRUE, FALSE)),
calc_time INTEGER, calc_time INTEGER,
bucket_1_name TEXT DEFAULT NULL,
bucket_2_name TEXT DEFAULT NULL,
bucket_3_name TEXT DEFAULT NULL,
bucket_4_name TEXT DEFAULT NULL,
bucket_5_name TEXT DEFAULT NULL,
bucket_6_name TEXT DEFAULT NULL,
bucket_7_name TEXT DEFAULT NULL,
bucket_8_name TEXT DEFAULT NULL,
bucket_9_name TEXT DEFAULT NULL,
comment TEXT DEFAULT NULL, comment TEXT DEFAULT NULL,
data TEXT NOT NULL, data TEXT NOT NULL,
ctime INTEGER NOT NULL DEFAULT (UNIXEPOCH()), ctime INTEGER NOT NULL DEFAULT (UNIXEPOCH()),
@ -661,16 +682,6 @@ CREATE TABLE payment_delivery_part (
mod_abs INTEGER NOT NULL DEFAULT 0, mod_abs INTEGER NOT NULL DEFAULT 0,
mod_rel REAL NOT NULL DEFAULT 0, mod_rel REAL NOT NULL DEFAULT 0,
bucket_1 INTEGER DEFAULT NULL,
bucket_2 INTEGER DEFAULT NULL,
bucket_3 INTEGER DEFAULT NULL,
bucket_4 INTEGER DEFAULT NULL,
bucket_5 INTEGER DEFAULT NULL,
bucket_6 INTEGER DEFAULT NULL,
bucket_7 INTEGER DEFAULT NULL,
bucket_8 INTEGER DEFAULT NULL,
bucket_9 INTEGER DEFAULT NULL,
price_1 INTEGER DEFAULT NULL, price_1 INTEGER DEFAULT NULL,
price_2 INTEGER DEFAULT NULL, price_2 INTEGER DEFAULT NULL,
price_3 INTEGER DEFAULT NULL, price_3 INTEGER DEFAULT NULL,

10
sql/v01/20.view.sql
View File

@ -23,7 +23,7 @@ SELECT p.year, p.did, p.dpnr,
d.date, d.time, d.zwstid, d.lnr, d.lsnr, d.date, d.time, d.zwstid, d.lnr, d.lsnr,
m.mgnr, m.family_name, m.given_name, m.mgnr, m.family_name, m.given_name,
p.sortid, p.weight, p.kmw, ROUND(p.kmw * (4.54 + 0.022 * p.kmw), 0) AS oe, p.qualid, p.hkid, p.kgnr, p.rdnr, p.sortid, p.weight, p.kmw, ROUND(p.kmw * (4.54 + 0.022 * p.kmw), 0) AS oe, p.qualid, p.hkid, p.kgnr, p.rdnr,
p.qualid NOT IN (SELECT l.qualid FROM wine_quality_level l WHERE p.kmw >= l.min_kmw OR l.min_kmw IS NULL) AS abgewertet, p.qualid IN (SELECT l.qualid FROM wine_quality_level l WHERE NOT l.predicate AND (p.kmw >= l.min_kmw OR l.min_kmw IS NULL) ORDER BY l.min_kmw DESC LIMIT 1,100) AS abgewertet,
p.qualid NOT IN ('WEI', 'RSW', 'LDW') AS min_quw, p.qualid NOT IN ('WEI', 'RSW', 'LDW') AS min_quw,
GROUP_CONCAT(DISTINCT a.attrid) as attributes, GROUP_CONCAT(DISTINCT o.modid) as modifiers, GROUP_CONCAT(DISTINCT a.attrid) as attributes, GROUP_CONCAT(DISTINCT o.modid) as modifiers,
d.comment, p.comment as part_comment d.comment, p.comment as part_comment
@ -35,13 +35,13 @@ FROM delivery_part p
GROUP BY p.year, p.did, p.dpnr GROUP BY p.year, p.did, p.dpnr
ORDER BY p.year, p.did, p.dpnr; ORDER BY p.year, p.did, p.dpnr;
CREATE VIEW v_bucket AS CREATE VIEW v_bin AS
SELECT year, mgnr, SELECT year, mgnr,
sortid || IIF(min_quw, REPLACE(COALESCE(attributes, ''), ',', ''), '_') AS bucket, sortid || IIF(min_quw, REPLACE(COALESCE(attributes, ''), ',', ''), '_') AS bin,
SUM(weight) AS weight SUM(weight) AS weight
FROM v_delivery FROM v_delivery
GROUP BY year, mgnr, bucket GROUP BY year, mgnr, bin
ORDER BY year, mgnr, LENGTH(bucket) DESC, bucket; ORDER BY year, mgnr, LENGTH(bin) DESC, bin;
CREATE VIEW v_stat_season AS CREATE VIEW v_stat_season AS
SELECT year, SELECT year,

2
sql/v01/99.schema_version.sql
View File

@ -1,3 +1,3 @@
-- This value MUST NOT be changed while other connections are open! -- This value MUST NOT be changed while other connections are open!
PRAGMA schema_version = 200; PRAGMA schema_version = 300;

2
wgmaster/import.py
View File

@ -15,7 +15,7 @@ DIR: str
TABLES = ['client_parameter', 'branch', 'wb_gl', 'wb_kg', 'wb_rd', 'wine_attribute', 'wine_cultivation', 'area_commitment_type', TABLES = ['client_parameter', 'branch', 'wb_gl', 'wb_kg', 'wb_rd', 'wine_attribute', 'wine_cultivation', 'area_commitment_type',
'member', 'member_billing_address', 'member_telephone_number', 'member_email_address', 'area_commitment', 'member', 'member_billing_address', 'member_telephone_number', 'member_email_address', 'area_commitment',
'season', 'modifier', 'delivery', 'delivery_part', 'delivery_part_attribute', 'delivery_part_modifier', 'season', 'modifier', 'delivery', 'delivery_part', 'delivery_part_attribute', 'delivery_part_modifier',
'payment_variant', 'payment_delivery_part'] 'delivery_part_bin', 'payment_variant', 'payment_delivery_part']
def get_sql_files() -> List[str]: def get_sql_files() -> List[str]:

31
wgmaster/migrate.py
View File

@ -1510,10 +1510,11 @@ def migrate_deliveries(in_dir: str, out_dir: str) -> None:
with utils.csv_open(f'{out_dir}/season.csv') as f_season, \ with utils.csv_open(f'{out_dir}/season.csv') as f_season, \
utils.csv_open(f'{out_dir}/modifier.csv') as f_mod: utils.csv_open(f'{out_dir}/modifier.csv') as f_mod:
f_season.header('year', 'currency', 'precision', 'start_date', 'end_date') f_season.header('year', 'currency', 'precision', 'start_date', 'end_date',
'bin_1_name', 'bin_2_name')
f_mod.header('year', 'modid', 'ordering', 'name', 'abs', 'rel', 'standard', 'quick_select') f_mod.header('year', 'modid', 'ordering', 'name', 'abs', 'rel', 'standard', 'quick_select')
for y, s in seasons.items(): for y, s in seasons.items():
f_season.row(y, s['currency'], s['precision'], s['start'], s['end']) f_season.row(y, s['currency'], s['precision'], s['start'], s['end'], 'gebunden', 'ungebunden')
for m in modifiers.values(): for m in modifiers.values():
abs_v = round(m['AZAS'] * pow(10, s['precision'])) if m['AZAS'] is not None else None abs_v = round(m['AZAS'] * pow(10, s['precision'])) if m['AZAS'] is not None else None
rel_v = m['AZASProzent'] / 100.0 if m['AZASProzent'] is not None else None rel_v = m['AZASProzent'] / 100.0 if m['AZASProzent'] is not None else None
@ -1533,8 +1534,7 @@ def migrate_payments(in_dir: str, out_dir: str) -> None:
qual_map = {i: [s for s in p_qual if s['AZNR'] == i] for i in set([s['AZNR'] for s in p_qual])} qual_map = {i: [s for s in p_qual if s['AZNR'] == i] for i in set([s['AZNR'] for s in p_qual])}
with utils.csv_open(f'{out_dir}/payment_variant.csv') as f_payment: with utils.csv_open(f'{out_dir}/payment_variant.csv') as f_payment:
f_payment.header('year', 'avnr', 'name', 'date', 'test_variant', 'calc_time', f_payment.header('year', 'avnr', 'name', 'date', 'test_variant', 'calc_time', 'comment', 'data')
'bucket_1_name', 'bucket_2_name', 'bucket_3_name', 'comment', 'data')
for p in utils.csv_parse_dict(f'{in_dir}/TAuszahlung.csv'): for p in utils.csv_parse_dict(f'{in_dir}/TAuszahlung.csv'):
year = p['Lesejahr'] year = p['Lesejahr']
if year is None: if year is None:
@ -1633,7 +1633,7 @@ def migrate_payments(in_dir: str, out_dir: str) -> None:
variant_year_map[year] = [] variant_year_map[year] = []
variant_year_map[year].append((p['AZNR'], year_map[year], p['TeilzahlungNr'])) variant_year_map[year].append((p['AZNR'], year_map[year], p['TeilzahlungNr']))
f_payment.row(year, year_map[year], p['Titel'], p['Datum'], test, None, f_payment.row(year, year_map[year], p['Titel'], p['Datum'], test, None,
'Gebunden', 'Nicht gebunden', 'Abgewertet', p['Beschreibung'], json.dumps(data)) p['Beschreibung'], json.dumps(data))
def get_modifiers(modifiers: str) -> Tuple[int, float]: def get_modifiers(modifiers: str) -> Tuple[int, float]:
if modifiers is None or modifiers == '': if modifiers is None or modifiers == '':
@ -1665,27 +1665,28 @@ def migrate_payments(in_dir: str, out_dir: str) -> None:
prec = pow(10, WGMASTER_PRECISION) prec = pow(10, WGMASTER_PRECISION)
return round(p1 * prec), round(p2 * prec), round(p3 * prec) return round(p1 * prec), round(p2 * prec), round(p3 * prec)
with utils.csv_open(f'{out_dir}/payment_delivery_part.csv') as f_del_pay: # TODO database migration
f_del_pay.header('year', 'did', 'dpnr', 'avnr', 'bucket_1', 'bucket_2', 'bucket_3', 'amount') with utils.csv_open(f'{out_dir}/payment_delivery_part.csv') as f_del_pay, \
utils.csv_open(f'{out_dir}/delivery_part_bin.csv') as f_bin:
f_del_pay.header('year', 'did', 'dpnr', 'avnr', 'amount')
f_bin.header('year', 'did', 'dpnr', 'bin_1', 'bin_2')
deliveries = {d['LINR']: d for d in utils.csv_parse_dict(f'{in_dir}/TLieferungen.csv')} deliveries = {d['LINR']: d for d in utils.csv_parse_dict(f'{in_dir}/TLieferungen.csv')}
for linr, (y, did, dpnr) in DELIVERY_MAP.items(): for linr, (y, did, dpnr) in DELIVERY_MAP.items():
p = deliveries[linr] p = deliveries[linr]
if y not in variant_year_map: if y not in variant_year_map:
continue continue
gew, geb_gew = int(p['Gewicht']), int(p['BGewichtGebunden'])
b2 = gew - geb_gew
b1 = geb_gew
f_bin.row(y, did, dpnr, b1, b2)
for aznr, avnr, tznr in variant_year_map[y]: for aznr, avnr, tznr in variant_year_map[y]:
val = p[f'BTeilzahlung{tznr}' if tznr < 6 else 'BEndauszahlung'] val = p[f'BTeilzahlung{tznr}' if tznr < 6 else 'BEndauszahlung']
val = round(val * pow(10, WGMASTER_PRECISION)) val = round(val * pow(10, WGMASTER_PRECISION))
b1, b2, b3 = 0, 0, 0
# prices = get_prices(aznr, p['SNR'], p['SANR'], int(p['Oechsle'])) # prices = get_prices(aznr, p['SNR'], p['SANR'], int(p['Oechsle']))
# mod = get_modifiers(p['BAbschlaegeString']) # mod = get_modifiers(p['BAbschlaegeString'])
# if not az_map[aznr].get('AbschlägeBerücksichtigen', False): # if not az_map[aznr].get('AbschlägeBerücksichtigen', False):
# mod = 0, 0.0 # mod = 0, 0.0
gew, geb_gew = int(p['Gewicht']), int(p['BGewichtGebunden'])
if QUAL_MAP[p['QSNR']] == 'WEI':
b3 += gew
else:
b2 += gew - geb_gew
b1 += geb_gew
# check_val = b1 * (prices[0] + mod[0]) + b2 * (prices[1] + mod[0]) + b3 * (prices[2] + mod[0]) # check_val = b1 * (prices[0] + mod[0]) + b2 * (prices[1] + mod[0]) + b3 * (prices[2] + mod[0])
# check_val *= 1 + mod[1] # check_val *= 1 + mod[1]
# check_val = round(check_val / 100) * 100 # check_val = round(check_val / 100) * 100
@ -1693,7 +1694,7 @@ def migrate_payments(in_dir: str, out_dir: str) -> None:
# print(p['LINR'], y, did, dpnr, avnr, val, check_val) # print(p['LINR'], y, did, dpnr, avnr, val, check_val)
# else: # else:
# print("GOOD") # print("GOOD")
f_del_pay.row(y, did, dpnr, avnr, b1, b2, b3, val) f_del_pay.row(y, did, dpnr, avnr, val)
def migrate_parameters(in_dir: str, out_dir: str) -> None: def migrate_parameters(in_dir: str, out_dir: str) -> None: