#!/usr/bin/env python3
import os, re, csv, json, shutil, subprocess
from pathlib import Path
from datetime import datetime

RAW=Path('/home/agent/jarvis_memory/work/_drive_sources/ERNE')
PROJECTS_RAW=RAW/'Projekte'
WISSEN_RAW=RAW/'ERNE_Wissen'
WORK=Path('/home/agent/jarvis_memory/work')
PROJECT_BASE=WORK/'projects'
WISSEN_BASE=WORK/'erne_wissen'

PROJECT_MAP={
 'Furkastrasse Schaffhausen':'furkastrasse',
 'Gemeindehaus Mönchaltorf':'moenchaltorf',
 'Schulanlage Hinterbirch Bülach':'schulanlage_hinterbirch_buelach',
}
MEDIA_EXT={'.mp4','.mov','.m4a','.mp3','.wav','.avi','.mkv'}
TEXT_EXT={'.txt','.md','.csv','.json'}
OFFICE_EXT={'.docx','.xlsx','.pptx'}
PDF_EXT={'.pdf'}

def rel_to(base,p): return p.relative_to(base).as_posix()

def esc(s): return str(s).replace('|','\\|').replace('\n',' ')

def ensure_link(src,dst):
    dst.parent.mkdir(parents=True, exist_ok=True)
    if dst.exists() or dst.is_symlink():
        try:
            if dst.is_symlink() and Path(os.readlink(dst))==src: return
            dst.unlink()
        except IsADirectoryError:
            return
    os.symlink(src,dst)

def extract_pdf(src,out):
    subprocess.run(['pdftotext','-layout',str(src),str(out)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=120)

def extract_docx(src,out):
    from docx import Document
    d=Document(str(src)); parts=[]
    for p in d.paragraphs:
        t=p.text.strip()
        if t: parts.append(t)
    for table in d.tables:
        for row in table.rows:
            cells=[c.text.strip().replace('\n',' ') for c in row.cells]
            if any(cells): parts.append(' | '.join(cells))
    out.write_text('\n'.join(parts), encoding='utf-8', errors='ignore')

def extract_xlsx(src,out):
    import openpyxl
    wb=openpyxl.load_workbook(src, data_only=True, read_only=True)
    lines=[]
    for ws in wb.worksheets:
        lines.append(f'# Sheet: {ws.title}')
        for row in ws.iter_rows(values_only=True):
            vals=[str(v) if v is not None else '' for v in row]
            if any(v.strip() for v in vals): lines.append('\t'.join(vals))
    out.write_text('\n'.join(lines), encoding='utf-8', errors='ignore')

def extract_pptx(src,out):
    from pptx import Presentation
    prs=Presentation(str(src)); lines=[]
    for i,slide in enumerate(prs.slides,1):
        lines.append(f'# Slide {i}')
        for shape in slide.shapes:
            if hasattr(shape,'text') and shape.text.strip(): lines.append(shape.text.strip())
    out.write_text('\n'.join(lines), encoding='utf-8', errors='ignore')

def extract_text(src,out):
    out.parent.mkdir(parents=True, exist_ok=True)
    if out.exists() and out.stat().st_mtime >= src.stat().st_mtime: return 'exists'
    ext=src.suffix.lower()
    try:
        if ext in PDF_EXT: extract_pdf(src,out)
        elif ext=='.docx': extract_docx(src,out)
        elif ext=='.xlsx': extract_xlsx(src,out)
        elif ext=='.pptx': extract_pptx(src,out)
        elif ext in TEXT_EXT: shutil.copyfile(src,out)
        else: return 'not_supported'
        if out.exists() and out.stat().st_size>0: return 'extracted'
        return 'empty_or_failed'
    except Exception as e:
        out.write_text(f'[EXTRACTION_ERROR] {type(e).__name__}: {e}\nSOURCE={src}\n', encoding='utf-8')
        return 'error'

def doc_type(rel):
    l=rel.lower()
    if 'werkvertrag' in l or 'avb' in l or 'vertrag' in l: return 'Werkvertrag/Vertrag'
    if 'termin' in l or 'bauprogramm' in l: return 'Terminplan'
    if 'pqm' in l or 'risik' in l or 'chance' in l or 'qs' in l or 'qualität' in l: return 'QS/PQM/Risiko'
    if 'protokoll' in l or 'sitzung' in l or 'bhs' in l or 'jour' in l: return 'Protokoll/Sitzung'
    if 'adress' in l or 'organigramm' in l: return 'Adressliste/Organisation'
    if 'bewilligung' in l: return 'Bewilligung'
    if 'nachtrag' in l or 'kosten' in l or 'zahlung' in l or 'budget' in l: return 'Kosten/Nachträge'
    if 'sia' in l or 'norm' in l: return 'Norm/Wissen'
    return 'Allgemein'

def build_collection(name, raw_base, out_base, project_slug=None):
    docs=out_base/'documents'
    links=docs/'source_files'
    texts=docs/'extracted_text'
    rows=[]
    for src in sorted([p for p in raw_base.rglob('*') if p.is_file()]):
        rel=rel_to(raw_base,src)
        ext=src.suffix.lower()
        is_media=ext in MEDIA_EXT
        link_path=links/rel
        ensure_link(src, link_path)
        text_rel=''
        ex_status='media_link_only' if is_media else 'not_supported'
        if not is_media:
            text_path=(texts/rel).with_suffix(src.suffix + '.txt')
            ex_status=extract_text(src,text_path)
            if text_path.exists(): text_rel=rel_to(docs,text_path)
        rows.append({
            'collection': name,
            'document': src.name,
            'type': doc_type(rel),
            'relative_path': rel,
            'source_link': rel_to(docs,link_path),
            'extracted_text': text_rel,
            'extension': ext,
            'bytes': src.stat().st_size,
            'extraction_status': ex_status,
        })
    docs.mkdir(parents=True, exist_ok=True)
    csv_path=docs/'document_index.csv'
    with csv_path.open('w',newline='',encoding='utf-8') as f:
        w=csv.DictWriter(f, fieldnames=list(rows[0].keys()) if rows else ['collection','document'])
        w.writeheader(); w.writerows(rows)
    md=docs/'README.md'
    bytype={}
    for r in rows: bytype[r['type']]=bytype.get(r['type'],0)+1
    lines=[f'# Dokumentenverzeichnis – {name}','',f'Erstellt/aktualisiert: {datetime.now().isoformat(timespec="seconds")}', '',
           f'- Quelle lokal: `{raw_base}`', f'- Dateien/Symlinks: `{links}`', f'- Suchbarer Textextrakt: `{texts}`', f'- Index CSV: `{csv_path}`','', '## Bestand nach Typ','']
    lines += [f'- {k}: {v}' for k,v in sorted(bytype.items())]
    lines += ['', '## Dokumente', '', '| Typ | Dokument | Pfad | Textextrakt |', '|---|---|---|---|']
    for r in rows:
        lines.append(f"| {esc(r['type'])} | {esc(r['document'])} | `{esc(r['source_link'])}` | `{esc(r['extracted_text'])}` |")
    md.write_text('\n'.join(lines)+'\n', encoding='utf-8')
    return rows, md, csv_path

summary=[]
for pname, slug in PROJECT_MAP.items():
    rb=PROJECTS_RAW/pname
    ob=PROJECT_BASE/slug
    ob.mkdir(parents=True, exist_ok=True)
    rows, md, csvp=build_collection(pname, rb, ob, slug)
    summary.append((pname, slug, len(rows), md))
    pm=ob/'project.md'
    insert=f"""\n\n## Dokumentenverzeichnis / Kontextquellen\n\nAktueller lokaler Dokumentenindex aus Google Drive ERNE/Projekte/{pname}:\n\n- Dokumentenverzeichnis: `documents/README.md`\n- Index CSV: `documents/document_index.csv`\n- Originaldateien als Symlinks: `documents/source_files/`\n- Suchbare Textextrakte: `documents/extracted_text/`\n\nNutzung: Bei Sitzungs-/Protokollvorbereitung zuerst `documents/README.md` und bei Detailfragen die passenden Textextrakte/Originale prüfen. Medienaufnahmen sind nur als Quelle/Datei abgelegt und werden nicht automatisch vollständig ausgewertet.\n"""
    if pm.exists():
        txt=pm.read_text(encoding='utf-8', errors='ignore')
        if '## Dokumentenverzeichnis / Kontextquellen' not in txt:
            pm.write_text(txt.rstrip()+insert+'\n', encoding='utf-8')
    else:
        pm.write_text(f"# {pname}\n"+insert, encoding='utf-8')

WISSEN_BASE.mkdir(parents=True, exist_ok=True)
wrows, wmd, wc=build_collection('ERNE_Wissen', WISSEN_RAW, WISSEN_BASE)
summary.append(('ERNE_Wissen','erne_wissen',len(wrows),wmd))

index=WORK/'ERNE_DOCUMENT_CONTEXT.md'
lines=['# ERNE Dokumentenkontext – Übersicht','',f'Aktualisiert: {datetime.now().isoformat(timespec="seconds")}', '', '| Sammlung | Slug | Dokumente | Index |','|---|---:|---:|---|']
for name,slug,n,md in summary:
    lines.append(f'| {esc(name)} | `{slug}` | {n} | `{md}` |')
index.write_text('\n'.join(lines)+'\n', encoding='utf-8')
print(json.dumps({'collections': [{'name':a,'slug':b,'documents':c,'index':str(d)} for a,b,c,d in summary], 'overview': str(index)}, ensure_ascii=False, indent=2))