Importing deliveries working

sql/v01/10.create.sql

@@ -417,16 +417,18 @@ CREATE TABLE modifier (
 ) STRICT;
 
 CREATE TABLE delivery (
-    year   INTEGER NOT NULL,
+    year    INTEGER NOT NULL,
-    did    INTEGER NOT NULL,
+    did     INTEGER NOT NULL,
 
-    date   TEXT    NOT NULL CHECK (date LIKE year || '-%' AND date REGEXP '^[1-9][0-9]{3}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$') DEFAULT CURRENT_DATE,
+    date    TEXT    NOT NULL CHECK (date LIKE year || '-%' AND date REGEXP '^[1-9][0-9]{3}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$') DEFAULT CURRENT_DATE,
-    time   TEXT    NOT NULL CHECK (time REGEXP '^([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]$')                                          DEFAULT CURRENT_TIME,
+    time    TEXT    NOT NULL CHECK (time REGEXP '^([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]$')                                          DEFAULT CURRENT_TIME,
-    zwstid TEXT    NOT NULL,
+    zwstid  TEXT    NOT NULL,
-    lnr    INTEGER NOT NULL CHECK (lnr >= 1 AND lnr <= 999),
+    lnr     INTEGER NOT NULL CHECK (lnr >= 1 AND lnr <= 999),
-    lsnr   TEXT    NOT NULL DEFAULT 'UNSET',
+    lsnr    TEXT    NOT NULL DEFAULT 'UNSET',
 
-    mgnr   INTEGER NOT NULL,
+    mgnr    INTEGER NOT NULL,
+
+    comment TEXT    DEFAULT NULL,
 
     CONSTRAINT pk_delivery PRIMARY KEY (year, did),
     CONSTRAINT sk_delivery_1 UNIQUE (date, zwstid, lnr),

wgmaster/import.py

@@ -5,12 +5,14 @@ import argparse
 import sqlite3
 import os
 import re
+import datetime
+
 import csv
 
 
 TABLES = ['branch', 'wb_gl', 'wb_kg', 'wb_rd', 'wine_attribute', 'wine_cultivation',
-          'member', 'member_billing_address', 'contract', 'area_commitment', ]
+          'member', 'member_billing_address', 'contract', 'area_commitment',
-# 'season', 'modifier', 'delivery', 'delivery_part', 'delivery_part_modifier',
+          'season', 'modifier', 'delivery', 'delivery_part', 'delivery_part_modifier', ]
 # 'payment_variant', 'delivery_payment', 'member_payment']
 
 
@@ -72,6 +74,9 @@ if __name__ == '__main__':
         except FileNotFoundError:
             pass
 
+    sqlite3.register_adapter(datetime.date, lambda d: str(d))
+    sqlite3.register_adapter(datetime.time, lambda t: str(t))
+
     DB_CNX = sqlite3.connect(args.db)
     DB_CNX.create_function('REGEXP', 2, sqlite_regexp)
 
@@ -83,6 +88,7 @@ if __name__ == '__main__':
 
     try:
         DB_CNX.isolation_level = None
+        # Member predecessors may refer to a higher MgNr
         DB_CNX.execute("PRAGMA foreign_keys = OFF")
         DB_CNX.execute("BEGIN")
         for table in TABLES:

wgmaster/migrate.py

@@ -1,6 +1,6 @@
 #!/bin/env python3
 
-from typing import Dict, Any, Tuple, Optional, List
+from typing import Dict, Any, Tuple, Optional, List, Iterable
 import argparse
 import os
 import re
@@ -241,6 +241,14 @@ def lookup_gem_name(name: str) -> List[Tuple[int, int]]:
     raise RuntimeError()
 
 
+def lookup_kg_name(kgnr: int) -> str:
+    cur = DB_CNX.cursor()
+    cur.execute("SELECT name FROM AT_kg WHERE kgnr = ?", (kgnr,))
+    rows = cur.fetchall()
+    cur.close()
+    return rows[0][0]
+
+
 def migrate_gradation(in_dir: str, out_dir: str) -> None:
     global GRADATION_MAP
     GRADATION_MAP = {}
@@ -339,7 +347,9 @@ def migrate_members(in_dir: str, out_dir: str) -> None:
     members = csv.parse(f'{in_dir}/TMitglieder.csv')
     fbs = parse_flaechenbindungen(in_dir)
 
-    with open(f'{out_dir}/member.csv', 'w+') as f_m, open(f'{out_dir}/member_billing_address.csv', 'w+') as f_mba:
+    with open(f'{out_dir}/member.csv', 'w+') as f_m,\
+            open(f'{out_dir}/member_billing_address.csv', 'w+') as f_mba,\
+            open(f'{out_dir}/wb_kg.csv', 'a') as f_kg:
         f_m.write('mgnr;predecessor_mgnr;prefix;given_name;middle_names;family_name;suffix;'
                   'birthday;entry_date;exit_date;business_shares;accounting_nr;zwstid;'
                   'lfbis_nr;ustid;volllieferant;buchführend;funktionär;active;iban;bic;'
@@ -544,6 +554,13 @@ def migrate_members(in_dir: str, out_dir: str) -> None:
             if kgnr is None:
                 invalid(mgnr, 'KGNr.', ort)
                 active = False
+            elif kgnr not in [kg[0] for gem in GEM_MAP.values() for kg in gem]:
+                glnr = list(GROSSLAGE_MAP.values())[0]
+                print(f'New KG: {lookup_kg_name(kgnr)} ({kgnr}, GL {glnr})')
+                f_kg.write(csv.format_row(kgnr, glnr))
+                if 9999 not in GEM_MAP:
+                    GEM_MAP[9999] = []
+                GEM_MAP[9999].append((kgnr, 0))
 
             if postal_dest is None:
                 invalid(mgnr, 'PLZ', None)
@@ -650,27 +667,76 @@ def migrate_contracts(in_dir: str, out_dir: str) -> None:
                 ))
 
 
+def fix_deliveries(deliveries: Iterable[Dict[str, Any]]) -> Iterable[Tuple[str, List[int], datetime.date]]:
+    dates = {}
+    fixed = {}
+    last_dates = {}
+
+    def add(lsnr: str, linr: int, date: datetime.date, unique: bool = False) -> None:
+        if lsnr not in fixed:
+            fixed[lsnr] = []
+            dates[lsnr] = date
+        elif unique:
+            return add(lsnr + '/2', linr, date, unique)
+        fixed[lsnr].append(linr)
+
+    def get_lsnr(date: datetime.date, lsnr: str) -> str:
+        if date.year < 2000:
+            return date.strftime('%y%m%d00') + lsnr[8:]
+        else:
+            return date.strftime('%Y%m%d') + lsnr[8:]
+
+    deliveries: List[Tuple[int, str, datetime.date, int, int]] = [
+        (d['LINR'], d['Lieferscheinnummer'], d['Datum'], d['ZNR'], d['MGNR'])
+        for d in deliveries
+        if d['Lieferscheinnummer'] and not d['Storniert']
+    ]
+
+    lsnrs = {d[1] for d in deliveries}
+
+    for lnr, lsnr, date, zwstnr, mgnr in deliveries:
+        lsdate = datetime.date(int(lsnr[:4]), int(lsnr[4:6]), int(lsnr[6:8])) if not lsnr.startswith('9') \
+            else datetime.date(1900 + int(lsnr[:2]), int(lsnr[2:4]), int(lsnr[4:6]))
+
+        if len(lsnr) == 12:
+            if date != lsdate:
+                if date.year == lsdate.year:
+                    lsnr_n = get_lsnr(date, lsnr)
+                    if lsnr_n not in lsnrs:
+                        print(f'{lsnr} -> {lsnr_n}')
+                        lsnr = lsnr_n
+                    else:
+                        warning_delivery(lsnr, mgnr, 'date', date)
+                else:
+                    date = datetime.date(lsdate.year, date.month, date.day)
+
+            if zwstnr not in last_dates or not date < last_dates[zwstnr]:
+                last_dates[zwstnr] = date
+            add(lsnr, lnr, date, unique=True)
+        else:
+            add(lsnr[:12], lnr, date)
+
+    return sorted([(f[0], f[1], dates[f[0]]) for f in fixed.items()],
+                  key=lambda f: f[0] if not f[0].startswith('9') else '19' + f[0])
+
+
 def migrate_deliveries(in_dir: str, out_dir: str) -> None:
     modifiers = {m['ASNR']: m for m in csv.parse(f'{in_dir}/TAbschlaege.csv') if m['Bezeichnung']}
     delivery_map = {}
     seasons = {}
-    comments: Dict[str, str] = {}
+    branches = {}
+
     deliveries = list(csv.parse(f'{in_dir}/TLieferungen.csv'))
+    delivery_dict = {d['LINR']: d for d in deliveries}
+    fixed = fix_deliveries(deliveries)
 
     with open(f'{out_dir}/delivery.csv', 'w+') as f_delivery, \
             open(f'{out_dir}/delivery_part.csv', 'w+') as f_part:
-        f_delivery.write('year;did;date;time;zwstid;lnr;lsnr;mgnr\n')
+        f_delivery.write('year;did;date;time;zwstid;lnr;lsnr;mgnr;comment\n')
         f_part.write('year;did;dpnr;sortid;attrid;weight;kmw;qualid;hkid;kgnr;rdnr;gerebelt;manual_weighing;spl_check;'
                      'hand_picked;lesemaschine;temperature;acid;scale_id;weighing_id;comment\n')
 
-        for d in deliveries:
+        for lsnr, linrs, date in fixed:
-            lsnr: str = d['Lieferscheinnummer']
-            if d['Storniert'] or lsnr is None:
-                if lsnr is not None:
-                    comments[lsnr] = d['Anmerkung']
-                continue
-
-            date: datetime.date = d['Datum']
             if date.year not in seasons:
                 seasons[date.year] = {
                     'currency': 'EUR' if date.year >= 2001 else 'ATS',
@@ -682,74 +748,78 @@ def migrate_deliveries(in_dir: str, out_dir: str) -> None:
             s = seasons[date.year]
             if date > s['end']:
                 s['end'] = date
-            snr, dpnr = 1, 1
+            s['nr'] += 1
-            comment: Optional[str] = d['Anmerkung']
+            snr = s['nr']
 
-            if lsnr[-1].isupper():
+            znr = delivery_dict[linrs[0]]['ZNR']
-                if lsnr[:-1] in comments:
+            if znr not in branches:
-                    comment = comments[lsnr[:-1]]
+                branches[znr] = {}
-                if lsnr[:-1] in delivery_map:
+            if date not in branches[znr]:
-                    d2 = delivery_map[lsnr[:-1]]
+                branches[znr][date] = 0
-                    snr = d2[1]
+            branches[znr][date] += 1
-                    dpnr = 2
+            lnr = branches[znr][date]
-                else:
+
-                    lsnr = lsnr[:-1]
+            comments = []
-            if not lsnr[-1].isupper():
+            for dpnr, linr in enumerate(linrs, start=1):
-                s['nr'] += 1
+                d = delivery_dict[linr]
-                snr = s['nr']
+                delivery_map[linr] = (date.year, snr, dpnr)
-                delivery_map[d['LINR']] = (date.year, snr)
+
-                delivery_map[lsnr] = delivery_map[d['LINR']]
+                oe = d['OechsleOriginal'] or d['Oechsle']
-                lnr = int(lsnr[9:12])
+                kmw = GRADATION_MAP[oe]
-                f_delivery.write(csv.format_row(
+                sortid, attrid = d['SNR'], d['SANR']
-                    date.year, snr, date, d['Uhrzeit'], BRANCH_MAP[d['ZNR']], lnr, lsnr, d['MGNR']
+                if len(sortid) != 2:
+                    attrid = sortid[-1]
+                    sortid = sortid[:2]
+                    print(f'{d["SNR"]} -> {sortid}/{attrid}')
+                kgnr, rdnr = None, None
+                if d['GNR']:
+                    gem = GEM_MAP[d['GNR']]
+                    if len(gem) == 1:
+                        kgnr = gem[0][0]
+                if d['RNR']:
+                    rd = REED_MAP[d['RNR']]
+                    # TODO reed nr
+                if kgnr is None:
+                    m = MEMBER_MAP[d['MGNR']]
+                    kgnr = m['default_kgnr']
+                if kgnr is None:
+                    warning_delivery(lsnr, d['MGNR'], 'KGNr.', None)
+                elif kgnr not in [kg[0] for gem in GEM_MAP.values() for kg in gem]:
+                    warning_delivery(lsnr, d['MGNR'], 'KGNr.', kgnr)
+                    kgnr = None
+
+                waage = d['Waagentext']
+                scale_id, weighing_id = None, None
+                if waage:
+                    waage = re.split(r' +', waage)
+                    scale_id = waage[1]
+                    weighing_id = waage[3]
+
+                comment: Optional[str] = d['Anmerkung']
+                acid = d['Säure']
+                hand, lesemaschine = None, None
+                if comment:
+                    comment = comment.replace('Söure', 'Säure')
+                    if comment.startswith('Säure'):
+                        acid = float(comment.split(' ')[-1].replace(',', '.'))
+                        comment = None
+                    elif comment == 'Maschine':
+                        hand = False
+                        comment = None
+                    elif comment == 'Hand':
+                        hand = True
+                        comment = None
+                if comment:
+                    comments.append(comment)
+
+                f_part.write(csv.format_row(
+                    date.year, snr, dpnr, sortid, attrid, int(d['Gewicht']), kmw, QUAL_MAP[d['QSNR']], HKID, kgnr, rdnr,
+                    d['Gerebelt'] or False, d['Handwiegung'] or False, d['Spaetlese-Ueberpruefung'] or False,
+                    hand, lesemaschine, d['Temperatur'], acid, scale_id, weighing_id, comment
                 ))
-
+            f_delivery.write(csv.format_row(
-            oe = d['OechsleOriginal'] or d['Oechsle']
+                date.year, snr, date, d['Uhrzeit'], BRANCH_MAP[d['ZNR']], lnr, lsnr, d['MGNR'],
-            kmw = GRADATION_MAP[oe]
+                '; '.join(comments) or None
-            sortid, attrid = d['SNR'], d['SANR']
-            if len(sortid) != 2:
-                attrid = sortid[-1]
-                sortid = sortid[:2]
-                print(f'{d["SNR"]} -> {sortid}/{attrid}')
-            kgnr, rdnr = None, None
-            if d['GNR']:
-                gem = GEM_MAP[d['GNR']]
-                if len(gem) == 1:
-                    kgnr = gem[0][0]
-            if d['RNR']:
-                rd = REED_MAP[d['RNR']]
-                # TODO reed nr
-            if kgnr is None:
-                m = MEMBER_MAP[d['MGNR']]
-                kgnr = m['default_kgnr']
-            if kgnr is None:
-                warning_delivery(lsnr, d['MGNR'], 'KGNr.', None)
-
-            waage = d['Waagentext']
-            scale_id, weighing_id = None, None
-            if waage:
-                waage = re.split(r' +', waage)
-                scale_id = waage[1]
-                weighing_id = waage[3]
-
-            acid = d['Säure']
-            hand, lesemaschine = None, None
-            if comment:
-                comment = comment.replace('Söure', 'Säure')
-                if comment.startswith('Säure'):
-                    acid = float(comment.split(' ')[-1].replace(',', '.'))
-                    comment = None
-                elif comment == 'Maschine':
-                    hand = False
-                    comment = None
-                elif comment == 'Hand':
-                    hand = True
-                    comment = None
-
-            f_part.write(csv.format_row(
-                date.year, snr, dpnr, sortid, attrid, int(d['Gewicht']), kmw, QUAL_MAP[d['QSNR']], HKID, kgnr, rdnr,
-                d['Gerebelt'] or False, d['Handwiegung'] or False, d['Spaetlese-Ueberpruefung'] or False,
-                hand, lesemaschine, d['Temperatur'], acid, scale_id, weighing_id, comment
             ))
 
     with open(f'{out_dir}/delivery_part_modifier.csv', 'w+') as f_part_mod:
@@ -758,7 +828,7 @@ def migrate_deliveries(in_dir: str, out_dir: str) -> None:
             if m['LINR'] not in delivery_map:
                 continue
             nid = delivery_map[m['LINR']]
-            f_part_mod.write(csv.format_row(nid[0], nid[1], 1, m['ASNR']))
+            f_part_mod.write(csv.format_row(nid[0], nid[1], nid[2], m['ASNR']))
 
     with open(f'{out_dir}/season.csv', 'w+') as f_season, open(f'{out_dir}/modifier.csv', 'w+') as f_mod:
         f_season.write('year;currency;precision;start_date;end_date\n')