iface_ecosio/flask_invoicing.py

from flask import Flask, request, render_template, send_file, render_template_string,abort, Response, redirect, url_for
from werkzeug.middleware.proxy_fix import ProxyFix
# from waitress import serve
import os
import json
import io
import re
import base64
import lzma
from io import BytesIO
from barcode import Code39 as BarCodeObj
from barcode.writer import ImageWriter
import qrcode
from datetime import date, timedelta, datetime
from xhtml2pdf import pisa
import zipfile
import datetime
from localconfig import *
import csv
from lib_poslog import process_invoice_request, etree_to_dict, get_trxlink_from_dict
import xml.etree.ElementTree as ET
from models import db, StateOfDate, StateOfInvoice
from pprint import pprint
from lib_poslog import process_invoice_request, etree_to_dict
from collections import deque


MAXZIPSIZE = 100  # max number of files in a zipfile to download


app = Flask(__name__)
app.config["SECRET_KEY"] = "lkjasfuhf83749yow8urehfkuresa"
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///state.db"

db.init_app(app)

with app.app_context():
    db.create_all()


@app.route('/invoice', methods=['GET'])
def home():
    return render_template('home.html')


@app.route('/invoice/download', methods=['GET'])
def download():
    # when all is done, read the ecosio folder and show all downloadable files, by country
    xmlfiles = {}   # structure: downloads[date][country]=filename
    dir_list = os.listdir(ECOSIO_PATH)
    for f in dir_list:
        if f.endswith('.xml'):   # PL_A282_T000_260110_1.xml
            country = f[0:2]  # first two chars
            date = f"20{f[13:19]}"
            if date not in xmlfiles:
                xmlfiles[date] = 0
            xmlfiles[date] += 1

    states = {}
    # r = StateOfDate.query.filter(system_issues=0, data_issues>0).all()
    date_states = StateOfDate.query.all()
    for d in date_states:
        if d.system_issues > 0 or d.data_issues > 0:
            files = 0
            if d.date in xmlfiles:
                files = xmlfiles[d.date]
            states[d.date] = [files, d.system_issues, d.data_issues]

    for yyyymmdd in xmlfiles.keys():
        if yyyymmdd not in states:
            states[yyyymmdd] = [xmlfiles[yyyymmdd],0,0]
        
    return render_template('ecosio_service.html', downloads=states)


@app.route('/invoice/downloadaszip', methods=['GET'])
def downloadaszip():
    # if there is a filename someone wants a download
    # if the filename is "all", then zip up all the avalable files and send the zip
    try:
        filenames = [
            f for f in os.listdir(ECOSIO_PATH)
            if f.endswith(".xml")
        ]
    except FileNotFoundError:
        abort(404, description="Files directory not found")

    if not filenames:
        abort(404, description="No xml files found")

    # Create in-memory ZIP file
    memory_file = io.BytesIO()

    with zipfile.ZipFile(memory_file, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
        for filename in filenames:  # PL_A282_T000_260110_1.xml
            file_path = os.path.join(ECOSIO_PATH, filename)
            zf.write(file_path, arcname=filename)
            # move the file to the archive
            os.replace(file_path, os.path.join(DONE_PATH, filename))
            # update db with file status TODO
            invoiceIsEcosiod(filename[3:].replace('.xml',''),0)  # parameter like A282_T000_260110_1

    memory_file.seek(0)

    return send_file(
        memory_file,
        mimetype="application/zip",
        as_attachment=True,
        download_name="bundle_for_ecosio.zip"
    )

def file_has_invoice(filename):
    is_invoice = False
    sto = dat = til = seq = ''
    transactiontytpe = '<RetailTransaction TransactionStatus="SES:Invoice">'
    with open(filename, 'r', encoding='utf-8') as f:
        while line := f.readline():
            if transactiontytpe in line:
                is_invoice = True
    if is_invoice:
        e = ET.parse(filename)
        root = e.getroot()
        tree = etree_to_dict(root)
        trx = tree['POSLog']['Transaction']['RetailTransaction']
        if 'InvoiceNumber' in trx:
            trx_link = get_trxlink_from_dict(trx)
            if len(trx_link) > 1:
                return f"{trx_link['store']}_T{trx_link['till']}_{trx_link['year']}{trx_link['month']}{trx_link['day']}_{trx_link['seq']}"
    return 0


@app.route('/invoice/requests', methods=['GET','POST'])
def get_invoices():
    '''
    this url takes a store and a date 
    then shows all found invoice-requests for that date
    clicking on one of these will show the invoice in message format
    '''
    xml_files = []
    store = ""
    date = ""

    if request.method == "POST":
        store = request.form.get("store")
        date = request.form.get("date")

        folder = os.path.join(SORTED_PATH, date, store)

        if folder and os.path.isdir(folder):
            for f in os.listdir(folder):
                if '_T000_' in f and f.endswith(".xml"):
                    filepath = os.path.join(folder, f)
                    i = file_has_invoice(filepath)
                    if i:
                        xml_files.append([f, i])

        else:
            xml_files = []

    return render_template(
        "requests.html",
        xml_files=xml_files,
        store=store,
        date=date
    )


@app.route("/invoice/invoice", methods=['GET'])
def view_invoice():
    '''
    this endpoint will show one invoice in message format
    '''
    date = request.args.get("date")
    store = request.args.get("store")
    filename = request.args.get("filename")

    if not date or not store or not filename:
        print("wrong url")
        abort(400)

    y,m,d = date.split('/')
    # req, sales = filename.split(':')
    filepath = os.path.join(SORTED_PATH, y, m, d, store, filename)

    # Basic safety check
    if not os.path.isfile(filepath) or not filepath.lower().endswith(".xml"):
        abort(404)

    invoice = process_invoice_request(filepath)
    return str(invoice)




def read_cust_csv():
    with open(BUYERS_BROKEN, newline="", encoding="utf-8") as f:
        reader = csv.reader(f)
        rows = list(reader)
    return rows[0], rows[1:]  # header, data


def write_repaired_csv(rows):
    with open(BUYERS_REPAIRED, "a", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        # writer.writerow(header)
        writer.writerows(rows)

def write_broken_csv(header, rows):
    print("write broken")
    with open(BUYERS_BROKEN, "w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow(header)
        writer.writerows(rows)

EDITABLECOLS = 7

@app.route("/invoice/buyers", methods=["GET", "POST"])
def broken_buyers():
    '''
    this endpoint will show all data from the broken-buyers csv file
    after upodating any line, if its approved as good  data, the buyer will move from broken to repaired csv
    '''
    header, rows = read_cust_csv()

    if request.method == "POST":
        updated_rows = []
        not_updated_rows = []

        for i, row in enumerate(rows):
            updated = row[:-EDITABLECOLS]  # untouched columns
            dataOk = True
            for j in range(EDITABLECOLS):
                field_name = f"row_{i}_col_{j}"
                value = request.form[field_name]
                if len(value) < 2:
                    dataOk = False
                updated.append(value)
            if dataOk:
                updated_rows.append(updated)
            else:
                not_updated_rows.append(updated)

        write_broken_csv(header, not_updated_rows)
        write_repaired_csv(updated_rows)
        return redirect(url_for("table"))

    return render_template(
        "buyers.html",
        header=header,
        rows=rows,
        editable_cols=EDITABLECOLS,
    )

@app.route('/invoice/api/getStatusOfDate/<YYYY>/<MM>/<DD>', methods=['GET'])
def getStatusOfDate(YYYY, MM, DD):
    # if there is a filename someone wants a download
    # if the filename is "all", then zip up all the avalable files and send the zip
    result = {}
    result['status'] = 'pending'
    if YYYY and MM and DD:
        d=f"{YYYY}{MM}{DD}"
        r = StateOfDate.query.filter_by(date=d).first()
        if r:
            result['data'] = str(r)
            if r.isDone():
                result['status'] = 'done'
    return json.dumps(result), 200


@app.route('/invoice/api/setStatusOfDate/<YYYY>/<MM>/<DD>/<systemerrors>/<dataerrors>', methods=['GET'])
def setStatusOfDate(YYYY, MM, DD, systemerrors, dataerrors):
    # if there is a filename someone wants a download
    # if the filename is "all", then zip up all the avalable files and send the zip
    result = {}
    if YYYY and MM and DD:
        d=f"{YYYY}{MM}{DD}"
        r = StateOfDate.query.filter_by(date=d).first()
        if r:
            r.system_issues = systemerrors
            r.data_issues = dataerrors
            db.session.commit()
            result['status'] = 'Ok'
            result['data'] = str(r)
        else:
            r = StateOfDate(date=f"{YYYY}{MM}{DD}", system_issues=systemerrors, data_issues=dataerrors)
            db.session.add(r)
            db.session.commit()
            result['status'] = 'Ok'
            result['data'] = str(r)

    return json.dumps(result), 200


@app.route('/invoice/api/getNextProcessDate/<YYYYMMDD>', methods=['GET'])
def getNextProcessDate(YYYYMMDD):
    result = {}
    latest_date = datetime.datetime.today().date()
    date = latest_date
    earliest_date = datetime.datetime.strptime(YYYYMMDD, "%Y%m%d").date()
    for sod in StateOfDate.query.all():
        if sod.system_issues > 0 or sod.data_issues > 0:
            date = datetime.datetime.strptime(sod.date, "%Y%m%d").date()
            if date >= earliest_date and date < latest_date:
                latest_date = date
    result['status'] = 'Ok'
    result['data'] = str(date.strftime('%Y%m%d'))

    return json.dumps(result), 200


def setInvoiceState(poslogid=None, receiptid=None, country='XX', state=-1):
    result = {}
    if poslogid:
        r = StateOfInvoice.query.filter_by(poslogid=poslogid).first()
        if r:
            r.state = state
            db.session.commit()
            result['status'] = 'Ok'
            result['data'] = str(r)
        else:
            YYYYMM=receiptid[4:10]
            print(f">->->{YYYYMM}")
            r = StateOfInvoice(poslogid=poslogid, receiptid=receiptid, receiptmonth=YYYYMM, country=country, state=state)
            db.session.add(r)
            db.session.commit()
            result['status'] = 'Ok'
            result['data'] = str(r)
    return json.dumps(result), 200


def getInvoiceState(poslogid=None):
    if poslogid:
        r = StateOfInvoice.query.filter_by(poslogid=poslogid).first()
        if r:
            return r.state
    return -1

@app.route('/invoice/api/poslog/isConverted/<poslogid>/<receiptid>/<country>', methods=['GET'])
def poslogIsConverted(poslogid, receiptid, country):
    # if there is a filename someone wants a download
    # if the filename is "all", then zip up all the avalable files and send the zip
    return setInvoiceState(poslogid=poslogid, receiptid=receiptid, country=country, state=1)

@app.route('/invoice/api/poslog/hasIssues/<poslogid>/<receiptid>/<country>', methods=['GET'])
def poslogHasIssues(poslogid, receiptid, country):
    # if there is a filename someone wants a download
    # if the filename is "all", then zip up all the avalable files and send the zip
    return setInvoiceState(poslogid=poslogid, receiptid=receiptid, country=country, state=0)

@app.route('/invoice/api/invoice/isEcosiod/<poslogid>', methods=['GET'])
def invoiceIsEcosiod(poslogid, receiptid):
    # if there is a filename someone wants a download
    # if the filename is "all", then zip up all the avalable files and send the zip
    return setInvoiceState(poslogid=poslogid, state=2)

@app.route('/invoice/api/invoice/isMonthlied/<poslogid>', methods=['GET'])
def isMonthlied(poslogid, receiptid):
    # if there is a filename someone wants a download
    # if the filename is "all", then zip up all the avalable files and send the zip
    return setInvoiceState(poslogid=poslogid, state=3)

@app.route('/invoice/api/poslog/canBeConverted/<poslogid>', methods=['GET'])
def isOkToCreateXML(poslogid):
    result = {}
    result['status'] = 'NotOk'
    # if there is a filename someone wants a download
    # if the filename is "all", then zip up all the avalable files and send the zip
    r = getInvoiceState(poslogid=poslogid)
    if r<2:
        result['status'] = 'Ok'
    return result

@app.route('/invoice/showtables/<status>', methods=['GET'])
def showPoslogs(status):
    # if there is a filename someone wants a download
    # if the filename is "all", then zip up all the avalable files and send the zip
    result = 'poslogs:<br/>'
    if status:
        poslogs = StateOfInvoice.query.all()
        if poslogs:
            for p in poslogs:
                result += f"{p}<br/>"

    result += '<br/>date-states:<br/>'   
    states = StateOfDate.query.all()
    if states:
        for p in states:
            result += f"{p}<br/>"
                
    return result

def file_has_invoicenr(filename, nrs):
    with open(filename, 'r', encoding='utf-8') as f:
        while line := f.readline():
            for nr in nrs:
                if nr in line:
                    return True
    return False


@app.route('/invoice/find_rejectedids', methods=['GET','POST'])
def find_rejectedids():
    rows=[]
    if request.method == "POST":
        if len(request.form['date']) == 6 and len(request.form['countrycode']) == 2 and  len(request.form['invoices']) > 10:
            # scroll though the ecosio/downloaded folder in search for <country>, later than <date>, then open them to see if they match an invoicenr
            invoices = []
            files = []
            for i in request.form['invoices'].split("\n"):
                if len(i) > 10:
                    invoices.append(i.strip())

            YY = request.form['date'][0:2]
            MM = request.form['date'][2:4]
            DD = request.form['date'][4:]

            for f in os.listdir(DONE_PATH):
                if f.startswith(f"{request.form['countrycode']}_") and f.endswith(".xml"):
                    country,store,till,filename_datestr,dummy = f.split('_')
                    file_date = datetime.datetime.strptime(filename_datestr, "%y%m%d").date()
                    ref_date = datetime.datetime.strptime(request.form['date'], "%y%m%d").date()
                    if file_date >= ref_date:
                        filepath = os.path.join(DONE_PATH, f)
                        if file_has_invoicenr(filepath, invoices):
                            # get the original sorted poslog file
                            filepath = os.path.join(SORTED_PATH, file_date.strftime("%Y"),file_date.strftime("%m"),file_date.strftime("%d"),store,f"{f[3:]}" )
                            inv = process_invoice_request(filepath)
                            row = [inv.country(), inv.nr, inv.receipt_id(), inv.poslogid, inv.buyer.name, inv.buyer.orgid, inv.buyer.taxID]
                            rows.append(row)

            if len(rows) > 0:
                return render_template("repair_rejectedids.html", rows=rows)
    return render_template("find_rejectedids.html")


@app.route('/invoice/repair_rejectedids', methods=['POST'])
def repair_rejectedids():
    files = {}
    download = []

    for f in request.form.keys():
        store, till, date, seq, field = f.split("_")
        filename = f"{store}_{till}_{date}_{seq}.xml"
        if not filename in files:
            files[filename] = {}
        files[filename][field] = request.form[f]

    # we now have a nice dict with filenames and their new orgid and taxid
    for f in files.keys():
        # make the sorted filename of the invoice:
        store, till, date, dummy = f.split("_")
        inv_filepath = os.path.join(SORTED_PATH, f"20{date[0:2]}", date[2:4], date[4:], store, f)
        inv = process_invoice_request(inv_filepath)
        # update the orgid and taxid
        inv.buyer.orgid = files[f]['orgid']
        inv.buyer.taxID = files[f]['taxid']
        # overwrite the file in the downloaded folder
        fname = f"{inv.country()}_{inv.poslogid}.xml"
        with open(os.path.join(DONE_PATH, fname), "w", encoding="utf-8") as lfile:
            lfile.write(str(inv))
            download.append(fname)

    # now zipo up and download the new files
    # Create in-memory ZIP file
    memory_file = io.BytesIO()

    with zipfile.ZipFile(memory_file, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
        for filename in download:  # PL_A282_T000_260110_1.xml
            file_path = os.path.join(DONE_PATH, filename)
            zf.write(file_path, arcname=filename)

    memory_file.seek(0)

    return send_file(
        memory_file,
        mimetype="application/zip",
        as_attachment=True,
        download_name="repairs_for_ecosio.zip"
    )

@app.route('/invoice/monthlyreport', methods=['GET','POST'])
def monthlyreport():
    csv = ''
    month = '202601'
    include = False
    if request.method == "POST":
        today = datetime.datetime.today().strftime('%Y-%m-%d')
        if len(request.form['countrycode']) == 2 and len(request.form['month']) == 6:
            country = request.form['countrycode']
            month = request.form['month']
            include = 'include_older' in request.form
            csv = 'ReportDate,InvoiceNr,InvoiceSentdate,oibSeller,oibBuyer,PaymentDate,PaymentAmount,PaymentMethod\n'
            if country == 'HR':
                csv = 'datumVrijemeSlanja,brojDokumenta,datumIzdavanja,oibPorezniBrojIzdavatelja,oibPorezniBrojPrimatelja,datumNaplate,naplaceniIznos,nacinPlacanja\n'
            # get the invoices from the database
            unreported = StateOfInvoice.query.filter_by(country=country, state=2).all()
            for inv in unreported:
                # get info from all 
                if (include and inv.receiptmonth <= month) or (inv.receiptmonth==month):
                    # load the ecosio file
                    filepath = os.path.join(DONE_PATH, f"{inv.country}_{inv.poslogid}.xml")
                    downloaddate = inv.updated_at.strftime('%Y-%m-%d')
                    if os.path.exists(filepath):
                        e = ET.parse(filepath)
                        root = e.getroot()
                        tree = etree_to_dict(root)
                        line = f"{today};{tree['Invoice']['ID']};{downloaddate};{tree['Invoice']['AccountingSupplierParty']['Party']['PartyLegalEntity']['CompanyID']};{tree['Invoice']['AccountingCustomerParty']['Party']['PartyLegalEntity']['CompanyID']};{tree['Invoice']['DueDate']};{tree['Invoice']['LegalMonetaryTotal']['PayableAmount']['#text']};T\n"
                        csv += line
                        isMonthlied(inv.poslogid, 0)
            # download to csv file
            buffer = io.BytesIO()
            buffer.write(csv.encode("utf-8"))
            buffer.seek(0)
            
            return send_file(
                buffer,
                mimetype="text/csv",
                as_attachment=True,
                download_name=f"monthly_report_{country}_{month}.csv"
            )

        return render_template("monthlyreport.html", )
    return render_template("monthlyreport.html")


def read_csv(csv_file):
    with open(csv_file, newline='', encoding="utf-8") as f:
        reader = list(csv.reader(f))
        header = reader[0]
        rows = reader[1:]
    return header, rows


def write_csv(csv_file, header, rows):
    with open(csv_file, "w", newline='', encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow(header)
        writer.writerows(rows)


@app.route("/invoice/maintain_csv/<csv_file>", methods=["GET", "POST"])
def maintain_csv(csv_file):
    csv_path = os.path.join(DATADIR, f"{csv_file}.csv")

    header, rows = read_csv(csv_path)

    if request.method == "POST":
        new_rows = []
        num_columns = len(header)

        # Collect form data
        row_index = 0
        while True:
            row = []
            for col_index in range(num_columns):
                field_name = f"row-{row_index}-col-{col_index}"
                if field_name not in request.form:
                    break
                row.append(request.form.get(field_name).strip())

            if len(row) != num_columns:
                break

            # Skip completely empty rows
            if any(cell != "" for cell in row):
                new_rows.append(row)

            row_index += 1

        write_csv(csv_path, header, new_rows)
        return redirect(url_for("home"))
    return render_template("edit_csv.html", header=header, rows=rows, csv=csv_file)


def tail(filename, n=500, loglevel=1):
    lines = []
    with open(filename, 'r', encoding='utf-8') as f:
        lines = list(deque(f, maxlen=n))
    if loglevel==0:
        return ''.join(lines)
    if loglevel==1:
        return ''.join( [ l for l in lines if LOGLEVELS[0] not in l ] )
    if loglevel>1:
        return ''.join( [ l for l in lines if LOGLEVELS[2] in l or LOGLEVELS[3] in l ] )


@app.route("/invoice/log/<loglevel>")
def show_log(loglevel):
    return render_template("show_log.html", txt=tail('log.txt', loglevel=int(loglevel)))

@app.route("/invoice/earliestdate", methods=["GET", "POST"])
def earliestdate():
    if request.method == "POST":
        earliestdate = request.form['earliestdate']
        with open(EARLIEST_DATE_FILE, 'w') as file:
            file.write(earliestdate)
    else:
        earliestdate = '20260101'
        if os.path.exists(EARLIEST_DATE_FILE):
            with open(EARLIEST_DATE_FILE, 'r') as f:
                earliestdate = f.readline()
    
    return render_template("update_earliest_date.html", earliestdate=earliestdate)


if __name__ == '__main__':
    # as native flask
    app.run(debug=True, host='0.0.0.0', port=9666)