winzer/elwig-misc
3
1
Fork 0
Code Pull Requests Activity

database: use bins instead of buckets

Browse Source
This commit is contained in:
Lorenz Stechauner
2023-10-11 23:43:45 +02:00
parent a70276adf8
commit 593eae7da4
5 changed files with 54 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
sql/v01/10.create.sql
View File

@ -444,6 +444,16 @@ CREATE TABLE season (
start_date TEXT CHECK (start_date REGEXP '^[1-9][0-9]{3}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$'),
end_date TEXT CHECK (end_date REGEXP '^[1-9][0-9]{3}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$'),
bin_1_name TEXT DEFAULT NULL,
bin_2_name TEXT DEFAULT NULL,
bin_3_name TEXT DEFAULT NULL,
bin_4_name TEXT DEFAULT NULL,
bin_5_name TEXT DEFAULT NULL,
bin_6_name TEXT DEFAULT NULL,
bin_7_name TEXT DEFAULT NULL,
bin_8_name TEXT DEFAULT NULL,
bin_9_name TEXT DEFAULT NULL,
CONSTRAINT pk_season PRIMARY KEY (year),
CONSTRAINT fk_season_currency FOREIGN KEY (currency) REFERENCES currency (code)
ON UPDATE CASCADE
@ -623,6 +633,27 @@ CREATE TABLE delivery_part_modifier (
ON DELETE RESTRICT
) STRICT;
CREATE TABLE delivery_part_bin (
year INTEGER NOT NULL,
did INTEGER NOT NULL,
dpnr INTEGER NOT NULL,
bin_1 INTEGER DEFAULT NULL,
bin_2 INTEGER DEFAULT NULL,
bin_3 INTEGER DEFAULT NULL,
bin_4 INTEGER DEFAULT NULL,
bin_5 INTEGER DEFAULT NULL,
bin_6 INTEGER DEFAULT NULL,
bin_7 INTEGER DEFAULT NULL,
bin_8 INTEGER DEFAULT NULL,
bin_9 INTEGER DEFAULT NULL,
CONSTRAINT pk_delivery_part_bin PRIMARY KEY (year, did, dpnr),
CONSTRAINT fk_delivery_part_bin_delivery_part FOREIGN KEY (year, did, dpnr) REFERENCES delivery_part (year, did, dpnr)
ON UPDATE CASCADE
ON DELETE CASCADE
) STRICT;
CREATE TABLE payment_variant (
year INTEGER NOT NULL,
avnr INTEGER NOT NULL,
@ -633,16 +664,6 @@ CREATE TABLE payment_variant (
test_variant INTEGER NOT NULL CHECK (test_variant IN (TRUE, FALSE)),
calc_time INTEGER,
bucket_1_name TEXT DEFAULT NULL,
bucket_2_name TEXT DEFAULT NULL,
bucket_3_name TEXT DEFAULT NULL,
bucket_4_name TEXT DEFAULT NULL,
bucket_5_name TEXT DEFAULT NULL,
bucket_6_name TEXT DEFAULT NULL,
bucket_7_name TEXT DEFAULT NULL,
bucket_8_name TEXT DEFAULT NULL,
bucket_9_name TEXT DEFAULT NULL,
comment TEXT DEFAULT NULL,
data TEXT NOT NULL,
ctime INTEGER NOT NULL DEFAULT (UNIXEPOCH()),
@ -661,16 +682,6 @@ CREATE TABLE payment_delivery_part (
mod_abs INTEGER NOT NULL DEFAULT 0,
mod_rel REAL NOT NULL DEFAULT 0,
bucket_1 INTEGER DEFAULT NULL,
bucket_2 INTEGER DEFAULT NULL,
bucket_3 INTEGER DEFAULT NULL,
bucket_4 INTEGER DEFAULT NULL,
bucket_5 INTEGER DEFAULT NULL,
bucket_6 INTEGER DEFAULT NULL,
bucket_7 INTEGER DEFAULT NULL,
bucket_8 INTEGER DEFAULT NULL,
bucket_9 INTEGER DEFAULT NULL,
price_1 INTEGER DEFAULT NULL,
price_2 INTEGER DEFAULT NULL,
price_3 INTEGER DEFAULT NULL,

10
sql/v01/20.view.sql
View File

@ -23,7 +23,7 @@ SELECT p.year, p.did, p.dpnr,
d.date, d.time, d.zwstid, d.lnr, d.lsnr,
m.mgnr, m.family_name, m.given_name,
p.sortid, p.weight, p.kmw, ROUND(p.kmw * (4.54 + 0.022 * p.kmw), 0) AS oe, p.qualid, p.hkid, p.kgnr, p.rdnr,
p.qualid NOT IN (SELECT l.qualid FROM wine_quality_level l WHERE p.kmw >= l.min_kmw OR l.min_kmw IS NULL) AS abgewertet,
p.qualid IN (SELECT l.qualid FROM wine_quality_level l WHERE NOT l.predicate AND (p.kmw >= l.min_kmw OR l.min_kmw IS NULL) ORDER BY l.min_kmw DESC LIMIT 1,100) AS abgewertet,
p.qualid NOT IN ('WEI', 'RSW', 'LDW') AS min_quw,
GROUP_CONCAT(DISTINCT a.attrid) as attributes, GROUP_CONCAT(DISTINCT o.modid) as modifiers,
d.comment, p.comment as part_comment
@ -35,13 +35,13 @@ FROM delivery_part p
GROUP BY p.year, p.did, p.dpnr
ORDER BY p.year, p.did, p.dpnr;
CREATE VIEW v_bucket AS
CREATE VIEW v_bin AS
SELECT year, mgnr,
sortid || IIF(min_quw, REPLACE(COALESCE(attributes, ''), ',', ''), '_') AS bucket,
sortid || IIF(min_quw, REPLACE(COALESCE(attributes, ''), ',', ''), '_') AS bin,
SUM(weight) AS weight
FROM v_delivery
GROUP BY year, mgnr, bucket
ORDER BY year, mgnr, LENGTH(bucket) DESC, bucket;
GROUP BY year, mgnr, bin
ORDER BY year, mgnr, LENGTH(bin) DESC, bin;
CREATE VIEW v_stat_season AS
SELECT year,

2
sql/v01/99.schema_version.sql
View File

@ -1,3 +1,3 @@
-- This value MUST NOT be changed while other connections are open!
PRAGMA schema_version = 200;
PRAGMA schema_version = 300;

2
wgmaster/import.py
View File

@ -15,7 +15,7 @@ DIR: str
TABLES = ['client_parameter', 'branch', 'wb_gl', 'wb_kg', 'wb_rd', 'wine_attribute', 'wine_cultivation', 'area_commitment_type',
'member', 'member_billing_address', 'member_telephone_number', 'member_email_address', 'area_commitment',
'season', 'modifier', 'delivery', 'delivery_part', 'delivery_part_attribute', 'delivery_part_modifier',
'payment_variant', 'payment_delivery_part']
'delivery_part_bin', 'payment_variant', 'payment_delivery_part']
def get_sql_files() -> List[str]:

31
wgmaster/migrate.py
View File

@ -1510,10 +1510,11 @@ def migrate_deliveries(in_dir: str, out_dir: str) -> None:
with utils.csv_open(f'{out_dir}/season.csv') as f_season, \
utils.csv_open(f'{out_dir}/modifier.csv') as f_mod:
f_season.header('year', 'currency', 'precision', 'start_date', 'end_date')
f_season.header('year', 'currency', 'precision', 'start_date', 'end_date',
'bin_1_name', 'bin_2_name')
f_mod.header('year', 'modid', 'ordering', 'name', 'abs', 'rel', 'standard', 'quick_select')
for y, s in seasons.items():
f_season.row(y, s['currency'], s['precision'], s['start'], s['end'])
f_season.row(y, s['currency'], s['precision'], s['start'], s['end'], 'gebunden', 'ungebunden')
for m in modifiers.values():
abs_v = round(m['AZAS'] * pow(10, s['precision'])) if m['AZAS'] is not None else None
rel_v = m['AZASProzent'] / 100.0 if m['AZASProzent'] is not None else None
@ -1533,8 +1534,7 @@ def migrate_payments(in_dir: str, out_dir: str) -> None:
qual_map = {i: [s for s in p_qual if s['AZNR'] == i] for i in set([s['AZNR'] for s in p_qual])}
with utils.csv_open(f'{out_dir}/payment_variant.csv') as f_payment:
f_payment.header('year', 'avnr', 'name', 'date', 'test_variant', 'calc_time',
'bucket_1_name', 'bucket_2_name', 'bucket_3_name', 'comment', 'data')
f_payment.header('year', 'avnr', 'name', 'date', 'test_variant', 'calc_time', 'comment', 'data')
for p in utils.csv_parse_dict(f'{in_dir}/TAuszahlung.csv'):
year = p['Lesejahr']
if year is None:
@ -1633,7 +1633,7 @@ def migrate_payments(in_dir: str, out_dir: str) -> None:
variant_year_map[year] = []
variant_year_map[year].append((p['AZNR'], year_map[year], p['TeilzahlungNr']))
f_payment.row(year, year_map[year], p['Titel'], p['Datum'], test, None,
'Gebunden', 'Nicht gebunden', 'Abgewertet', p['Beschreibung'], json.dumps(data))
p['Beschreibung'], json.dumps(data))
def get_modifiers(modifiers: str) -> Tuple[int, float]:
if modifiers is None or modifiers == '':
@ -1665,27 +1665,28 @@ def migrate_payments(in_dir: str, out_dir: str) -> None:
prec = pow(10, WGMASTER_PRECISION)
return round(p1 * prec), round(p2 * prec), round(p3 * prec)
with utils.csv_open(f'{out_dir}/payment_delivery_part.csv') as f_del_pay:
f_del_pay.header('year', 'did', 'dpnr', 'avnr', 'bucket_1', 'bucket_2', 'bucket_3', 'amount')
# TODO database migration
with utils.csv_open(f'{out_dir}/payment_delivery_part.csv') as f_del_pay, \
utils.csv_open(f'{out_dir}/delivery_part_bin.csv') as f_bin:
f_del_pay.header('year', 'did', 'dpnr', 'avnr', 'amount')
f_bin.header('year', 'did', 'dpnr', 'bin_1', 'bin_2')
deliveries = {d['LINR']: d for d in utils.csv_parse_dict(f'{in_dir}/TLieferungen.csv')}
for linr, (y, did, dpnr) in DELIVERY_MAP.items():
p = deliveries[linr]
if y not in variant_year_map:
continue
gew, geb_gew = int(p['Gewicht']), int(p['BGewichtGebunden'])
b2 = gew - geb_gew
b1 = geb_gew
f_bin.row(y, did, dpnr, b1, b2)
for aznr, avnr, tznr in variant_year_map[y]:
val = p[f'BTeilzahlung{tznr}' if tznr < 6 else 'BEndauszahlung']
val = round(val * pow(10, WGMASTER_PRECISION))
b1, b2, b3 = 0, 0, 0
# prices = get_prices(aznr, p['SNR'], p['SANR'], int(p['Oechsle']))
# mod = get_modifiers(p['BAbschlaegeString'])
# if not az_map[aznr].get('AbschlägeBerücksichtigen', False):
# mod = 0, 0.0
gew, geb_gew = int(p['Gewicht']), int(p['BGewichtGebunden'])
if QUAL_MAP[p['QSNR']] == 'WEI':
b3 += gew
else:
b2 += gew - geb_gew
b1 += geb_gew
# check_val = b1 * (prices[0] + mod[0]) + b2 * (prices[1] + mod[0]) + b3 * (prices[2] + mod[0])
# check_val *= 1 + mod[1]
# check_val = round(check_val / 100) * 100
@ -1693,7 +1694,7 @@ def migrate_payments(in_dir: str, out_dir: str) -> None:
# print(p['LINR'], y, did, dpnr, avnr, val, check_val)
# else:
# print("GOOD")
f_del_pay.row(y, did, dpnr, avnr, b1, b2, b3, val)
f_del_pay.row(y, did, dpnr, avnr, val)
def migrate_parameters(in_dir: str, out_dir: str) -> None: