winzer/elwig-misc
3
1
Fork 0
Code Pull Requests Activity

Migrate deliveries

Browse Source
This commit is contained in:
Lorenz Stechauner
2023-04-02 15:55:10 +02:00
parent 7f908a2966
commit fef88f2f20
3 changed files with 199 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
sql/v01/10.create.sql
View File

@ -249,8 +249,8 @@ CREATE TABLE branch (
);
CREATE TABLE wine_attribute (
attrid TEXT NOT NULL CHECK (attrid REGEXP '^[A-Z]+$'),
name TEXT NOT NULL,
attrid TEXT NOT NULL CHECK (attrid REGEXP '^[A-Z]+$'),
name TEXT NOT NULL,
kg_per_ha INTEGER NOT NULL DEFAULT 10000,
@ -444,7 +444,7 @@ CREATE TABLE delivery (
CREATE TRIGGER t_delivery_i
AFTER INSERT ON delivery FOR EACH ROW
WHEN NEW.lsnr = 'UNSET'
WHEN NEW.lsnr = 'UNSET'
BEGIN
UPDATE delivery
SET lsnr = format('%04s%02s%02s%1s%03i', substr(NEW.date, 1, 4), substr(NEW.date, 6, 2), substr(NEW.date, 9, 2), zwstid, lnr)
@ -452,29 +452,35 @@ BEGIN
END;
CREATE TABLE delivery_part (
year INTEGER NOT NULL,
did INTEGER NOT NULL,
dpnr INTEGER NOT NULL,
year INTEGER NOT NULL,
did INTEGER NOT NULL,
dpnr INTEGER NOT NULL,
sortid TEXT NOT NULL,
attrid TEXT DEFAULT NULL,
sortid TEXT NOT NULL,
attrid TEXT DEFAULT NULL,
weight INTEGER NOT NULL,
kmw REAL NOT NULL,
qualid TEXT NOT NULL,
weight INTEGER NOT NULL,
kmw REAL NOT NULL,
qualid TEXT NOT NULL,
hkid TEXT NOT NULL,
kgnr INTEGER DEFAULT NULL,
rdnr INTEGER DEFAULT NULL,
hkid TEXT NOT NULL,
kgnr INTEGER DEFAULT NULL,
rdnr INTEGER DEFAULT NULL,
gerebelt INTEGER NOT NULL CHECK (gerebelt IN (TRUE, FALSE)),
handwiegung INTEGER NOT NULL CHECK (handwiegung IN (TRUE, FALSE)),
spätleseüberprüfung INTEGER NOT NULL CHECK (spätleseüberprüfung IN (TRUE, FALSE)) DEFAULT FALSE,
gerebelt INTEGER NOT NULL CHECK (gerebelt IN (TRUE, FALSE)),
manual_weighing INTEGER NOT NULL CHECK (manual_weighing IN (TRUE, FALSE)),
spl_check INTEGER NOT NULL CHECK (spl_check IN (TRUE, FALSE)) DEFAULT FALSE,
temperature REAL DEFAULT NULL,
acid REAL DEFAULT NULL,
comment TEXT DEFAULT NULL,
waagentext TEXT,
hand_picked INTEGER CHECK (hand_picked IN (TRUE, FALSE)) DEFAULT NULL,
lesemaschine INTEGER CHECK (lesemaschine IN (True, FALSE)) DEFAULT NULL,
temperature REAL DEFAULT NULL,
acid REAL DEFAULT NULL,
scale_id TEXT,
weighing_id TEXT,
comment TEXT DEFAULT NULL,
CONSTRAINT pk_delivery_part PRIMARY KEY (year, did, dpnr),
CONSTRAINT fk_delivery_part_delivery FOREIGN KEY (year, did) REFERENCES delivery (year, did)
@ -502,7 +508,7 @@ CREATE TABLE delivery_part (
CREATE TRIGGER t_delivery_part_i
AFTER INSERT ON delivery_part FOR EACH ROW
WHEN NEW.kgnr IS NOT NULL
WHEN NEW.kgnr IS NOT NULL
BEGIN
UPDATE delivery_part SET hkid = (
SELECT hkid
@ -515,7 +521,7 @@ END;
CREATE TRIGGER t_delivery_part_u
AFTER UPDATE OF kgnr ON delivery_part FOR EACH ROW
WHEN NEW.kgnr IS NOT NULL
WHEN NEW.kgnr IS NOT NULL
BEGIN
UPDATE delivery_part SET hkid = (
SELECT hkid

6
wgmaster/csv.py
View File

@ -47,12 +47,14 @@ def parse(filename: str) -> Iterator[Dict[str, Any]]:
part = False
elif part.isdigit():
part = int(part)
elif re.match(r'[0-9]+\.[0-9]+', part):
elif re.match(r'-?[0-9]+\.[0-9]+', part):
part = float(part)
elif len(part) == 10 and part[4] == '-' and part[7] == '-':
part = datetime.datetime.strptime(part, '%Y-%m-%d').date()
elif len(part) == 8 and part[2] == ':' and part[5] == ':':
part = datetime.time.fromisoformat(part)
else:
raise RuntimeError(part)
raise RuntimeError(f'unable to infer type of value "{part}"')
obj[header[i]] = part
yield obj

166
wgmaster/migrate.py
View File

@ -7,21 +7,36 @@ import re
import sys
import sqlite3
import requests
import datetime
import csv
DB_CNX: Optional[sqlite3.Connection] = None
HKID: Optional[str] = None
USTID_RE = re.compile(r'[A-Z]{2}[A-Z0-9]{2,12}')
BIC_RE = re.compile(r'[A-Z0-9]{4}[A-Z]{2}[A-Z0-9]{2}([A-Z0-9]{3})?')
IBAN_RE = re.compile(r'[A-Z]{2}[0-9]{2}[A-Z0-9]{8,30}')
EMAIL_RE = re.compile(r'[^@\s]+@([a-z0-9_äöüß-]+\.)+[a-z]{2,}')
GRADATION_MAP: Optional[Dict[float, float]] = None
CULTIVATION_MAP: Optional[Dict[int, str]] = None
BRANCH_MAP: Optional[Dict[int, str]] = None
GEM_MAP: Optional[Dict[int, List[Tuple[int, int]]]] = None
REED_MAP: Optional[Dict[int, Tuple[int, int]]] = None
GROSSLAGE_MAP: Optional[Dict[int, int]] = None
MEMBER_MAP: Optional[Dict[int, Dict[str, Any]]] = None
QUAL_MAP: Dict[int, str] = {
0: 'WEI',
1: 'RSW',
2: 'LDW',
3: 'QUW',
4: 'KAB',
5: 'SPL',
}
STREET_NAMES: Dict[str, str] = {
'Hans-Wagnerstraße': 'Hans-Wagner-Straße',
@ -73,6 +88,14 @@ def invalid(mgnr: int, key: str, value: str) -> None:
print(f'\x1B[1;31m{mgnr:>6}: {key:<12} {value}\x1B[0m', file=sys.stderr)
def warning_delivery(lsnr: str, mgnr: int, key:str, value: str) -> None:
print(f'\x1B[1;33m{lsnr:<13} ({mgnr:>6}): {key:<12} {value}\x1B[0m', file=sys.stderr)
def invalid_delivery(lsnr: str, mgnr: int, key: str, value: str) -> None:
print(f'\x1B[1;31m{lsnr:<13} ({mgnr:>6}): {key:<12} {value}\x1B[0m', file=sys.stderr)
def convert(mgnr: int, key: str, old_value: str, new_value: str) -> None:
if not args.quiet:
print(f'\x1B[1m{mgnr:>6}: {key:<12} "{old_value}" -> "{new_value}"\x1B[0m', file=sys.stderr)
@ -218,6 +241,13 @@ def lookup_gem_name(name: str) -> List[Tuple[int, int]]:
raise RuntimeError()
def migrate_gradation(in_dir: str, out_dir: str) -> None:
global GRADATION_MAP
GRADATION_MAP = {}
for g in csv.parse(f'{in_dir}/TUmrechnung.csv'):
GRADATION_MAP[g['Oechsle']] = g['KW']
def migrate_branches(in_dir: str, out_dir: str) -> None:
global BRANCH_MAP
BRANCH_MAP = {}
@ -303,6 +333,9 @@ def migrate_cultivations(in_dir: str, out_dir: str) -> None:
def migrate_members(in_dir: str, out_dir: str) -> None:
global MEMBER_MAP
MEMBER_MAP = {}
members = csv.parse(f'{in_dir}/TMitglieder.csv')
fbs = parse_flaechenbindungen(in_dir)
@ -525,6 +558,9 @@ def migrate_members(in_dir: str, out_dir: str) -> None:
phone_mobile[0] if len(phone_mobile) > 0 else None, phone_mobile[1] if len(phone_mobile) > 1 else None,
kgnr, m['Anmerkung']
))
MEMBER_MAP[mgnr] = {
'default_kgnr': kgnr
}
if billing_name:
f_mba.write(csv.format_row(mgnr, billing_name, 'AT', postal_dest, address or '-'))
@ -614,6 +650,129 @@ def migrate_contracts(in_dir: str, out_dir: str) -> None:
))
def migrate_deliveries(in_dir: str, out_dir: str) -> None:
modifiers = {m['ASNR']: m for m in csv.parse(f'{in_dir}/TAbschlaege.csv') if m['Bezeichnung']}
delivery_map = {}
seasons = {}
comments = {}
with open(f'{out_dir}/delivery.csv', 'w+') as f_delivery, \
open(f'{out_dir}/delivery_part.csv', 'w+') as f_part:
f_delivery.write('year;did;date;time;zwstid;lnr;lsnr;mgnr\n')
f_part.write('year;did;dpnr;sortid;attrid;weight;kmw;qualid;hkid;kgnr;rdnr;gerebelt;manual_weighing;spl_check;'
'hand_picked;lesemaschine;temperature;acid;scale_id;weighing_id;comment\n')
for d in sorted(csv.parse(f'{in_dir}/TLieferungen.csv'), key=lambda l: f'{l["Datum"]}T{l["Uhrzeit"]}'):
lsnr: str = d['Lieferscheinnummer']
if d['Storniert'] or lsnr is None:
comments[lsnr] = d['Anmerkung']
continue
date: datetime.date = d['Datum']
if date.year not in seasons:
seasons[date.year] = {
'currency': 'EUR' if date.year >= 2001 else 'ATS',
'precision': 4,
'start': date,
'end': date,
'nr': 0,
}
s = seasons[date.year]
if date > s['end']:
s['end'] = date
snr, dpnr = 1, 1
comment: Optional[str] = d['Anmerkung']
if lsnr.endswith('A'):
if lsnr[:-1] in comments:
comment = comments[lsnr[:-1]]
if lsnr[:-1] in delivery_map:
d2 = delivery_map[lsnr[:-1]]
snr = d2[1]
dpnr = 2
else:
lsnr = lsnr[:-1]
if not lsnr.endswith('A'):
s['nr'] += 1
snr = s['nr']
delivery_map[d['LINR']] = (date.year, snr)
delivery_map[lsnr] = delivery_map[d['LINR']]
lnr = int(lsnr[9:12])
f_delivery.write(csv.format_row(
date.year, snr, date, d['Uhrzeit'], BRANCH_MAP[d['ZNR']], lnr, lsnr, d['MGNR']
))
oe = d['OechsleOriginal'] or d['Oechsle']
kmw = GRADATION_MAP[oe]
sortid, attrid = d['SNR'], d['SANR']
if len(sortid) != 2:
attrid = sortid[-1]
sortid = sortid[:2]
print(f'{d["SNR"]} -> {sortid}/{attrid}')
kgnr, rdnr = None, None
if d['GNR']:
gem = GEM_MAP[d['GNR']]
if len(gem) == 1:
kgnr = gem[0][0]
if d['RNR']:
rd = REED_MAP[d['RNR']]
# TODO reed nr
if kgnr is None:
m = MEMBER_MAP[d['MGNR']]
kgnr = m['default_kgnr']
if kgnr is None:
warning_delivery(lsnr, d['MGNR'], 'KgNr.', None)
waage = d['Waagentext']
scale_id, weighing_id = None, None
if waage:
waage = re.split(r' +', waage)
scale_id = int(waage[1])
weighing_id = int(waage[3])
acid = d['Säure']
hand, lesemaschine = None, None
if comment:
comment = comment.replace('Söure', 'Säure')
if comment.startswith('Säure'):
acid = float(comment.split(' ')[-1].replace(',', '.'))
comment = None
elif comment == 'Maschine':
hand = False
comment = None
elif comment == 'Hand':
hand = True
comment = None
f_part.write(csv.format_row(
date.year, snr, dpnr, sortid, attrid, int(d['Gewicht']), kmw, QUAL_MAP[d['QSNR']], HKID, kgnr, rdnr,
d['Gerebelt'] or False, d['Handwiegung'] or False, d['Spaetlese-Ueberpruefung'] or False,
hand, lesemaschine, d['Temperatur'], acid, scale_id, weighing_id, comment
))
with open(f'{out_dir}/delivery_part_modifier.csv', 'w+') as f_part_mod:
f_part_mod.write('year;did;dpnr;mnr\n')
for m in csv.parse(f'{in_dir}/TLieferungAbschlag.csv'):
if m['LINR'] not in delivery_map:
continue
nid = delivery_map[m['LINR']]
f_part_mod.write(csv.format_row(nid[0], nid[1], 1, m['ASNR']))
with open(f'{out_dir}/season.csv', 'w+') as f_season, open(f'{out_dir}/modifier.csv', 'w+') as f_mod:
f_season.write('year;currency;precision;start_date;end_date\n')
f_mod.write('year;mnr;name;abs;rel;standard;quick_select\n')
for y, s in seasons.items():
f_season.write(csv.format_row(y, s['currency'], s['precision'], s['start'], s['end']))
for m in modifiers.values():
f_mod.write(csv.format_row(
y, m['ASNR'], m['Bezeichnung'], m['AZAS'], m['AZASProzent'], m['Standard'], m['Schnellauswahl']
))
def migrate_payments(in_dir: str, out_dir: str) -> None:
pass # TODO migrate payments
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('in_dir', type=str,
@ -624,12 +783,17 @@ if __name__ == '__main__':
help='Be less verbose')
parser.add_argument('-d', '--database', metavar='DB', required=True,
help='The sqlite database file to look up information')
parser.add_argument('-o', '--origin', metavar='HKID', required=True,
help='The default wine origin identifier')
args = parser.parse_args()
os.makedirs(args.out_dir, exist_ok=True)
HKID = args.origin
DB_CNX = sqlite3.connect(args.database)
migrate_gradation(args.in_dir, args.out_dir)
migrate_branches(args.in_dir, args.out_dir)
migrate_grosslagen(args.in_dir, args.out_dir)
migrate_gemeinden(args.in_dir, args.out_dir)
@ -638,5 +802,7 @@ if __name__ == '__main__':
migrate_cultivations(args.in_dir, args.out_dir)
migrate_members(args.in_dir, args.out_dir)
migrate_contracts(args.in_dir, args.out_dir)
migrate_deliveries(args.in_dir, args.out_dir)
migrate_payments(args.in_dir, args.out_dir)
DB_CNX.close()