tons of stuff but working again
This commit is contained in:
@ -35,7 +35,7 @@ from plugins.plant.models import (
|
||||
)
|
||||
from plugins.media.models import Media
|
||||
from plugins.utility.models import ImportBatch
|
||||
from plugins.media import _process_upload_file
|
||||
from plugins.media.routes import _process_upload_file
|
||||
|
||||
bp = Blueprint(
|
||||
'utility',
|
||||
@ -79,6 +79,9 @@ REQUIRED_HEADERS = {"uuid", "plant_type", "name", "scientific_name", "mother_uui
|
||||
@bp.route("/upload", methods=["GET", "POST"])
|
||||
@login_required
|
||||
def upload():
|
||||
# -------------------------------------------------------------------------
|
||||
# POST handling
|
||||
# -------------------------------------------------------------------------
|
||||
if request.method == "POST":
|
||||
file = request.files.get("file")
|
||||
if not file or not file.filename:
|
||||
@ -89,44 +92,46 @@ def upload():
|
||||
|
||||
# ── ZIP Import Flow ────────────────────────────────────────────────
|
||||
if filename.endswith(".zip"):
|
||||
tmp_zip = tempfile.NamedTemporaryFile(delete=False, suffix=".zip")
|
||||
file.save(tmp_zip.name)
|
||||
tmp_zip.close()
|
||||
# 1) Save to shared UPLOAD_FOLDER
|
||||
upload_dir = current_app.config["UPLOAD_FOLDER"]
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
tmp_path = os.path.join(upload_dir, f"tmp_{uuid.uuid4().hex}.zip")
|
||||
file.save(tmp_path)
|
||||
|
||||
# validate ZIP
|
||||
# 2) Validate ZIP + contents
|
||||
try:
|
||||
z = zipfile.ZipFile(tmp_zip.name)
|
||||
zf = zipfile.ZipFile(tmp_path)
|
||||
except zipfile.BadZipFile:
|
||||
os.remove(tmp_zip.name)
|
||||
os.remove(tmp_path)
|
||||
flash("Uploaded file is not a valid ZIP.", "danger")
|
||||
return redirect(request.url)
|
||||
|
||||
names = z.namelist()
|
||||
if "plants.csv" not in names or "media.csv" not in names:
|
||||
os.remove(tmp_zip.name)
|
||||
names = set(zf.namelist())
|
||||
if not {"plants.csv", "media.csv"}.issubset(names):
|
||||
os.remove(tmp_path)
|
||||
flash("ZIP must contain both plants.csv and media.csv", "danger")
|
||||
return redirect(request.url)
|
||||
|
||||
# extract export_id from metadata
|
||||
# 3) Extract export_id from metadata.txt
|
||||
export_id = None
|
||||
if "metadata.txt" in names:
|
||||
meta = z.read("metadata.txt").decode("utf-8", "ignore")
|
||||
meta = zf.read("metadata.txt").decode("utf-8", "ignore")
|
||||
for line in meta.splitlines():
|
||||
if line.startswith("export_id,"):
|
||||
export_id = line.split(",", 1)[1].strip()
|
||||
break
|
||||
if not export_id:
|
||||
os.remove(tmp_zip.name)
|
||||
os.remove(tmp_path)
|
||||
flash("metadata.txt missing or missing export_id", "danger")
|
||||
return redirect(request.url)
|
||||
|
||||
# prevent duplicates
|
||||
# 4) Prevent duplicate imports
|
||||
if ImportBatch.query.filter_by(export_id=export_id, user_id=current_user.id).first():
|
||||
os.remove(tmp_zip.name)
|
||||
os.remove(tmp_path)
|
||||
flash("This export has already been imported.", "info")
|
||||
return redirect(request.url)
|
||||
|
||||
# record batch
|
||||
# 5) Create batch record
|
||||
batch = ImportBatch(
|
||||
export_id = export_id,
|
||||
user_id = current_user.id,
|
||||
@ -136,174 +141,15 @@ def upload():
|
||||
db.session.add(batch)
|
||||
db.session.commit()
|
||||
|
||||
# hand off to Celery
|
||||
try:
|
||||
import_text_data.delay(tmp_zip.name, "zip", batch.id)
|
||||
flash("ZIP received; import queued in background.", "success")
|
||||
return redirect(request.url)
|
||||
except Exception:
|
||||
current_app.logger.exception("Failed to enqueue import_text_data")
|
||||
flash("Failed to queue import job; falling back to inline import", "warning")
|
||||
# 6) Rename ZIP to include batch.id
|
||||
final_path = os.path.join(upload_dir, f"{batch.id}_{uuid.uuid4().hex}.zip")
|
||||
os.rename(tmp_path, final_path)
|
||||
|
||||
# ── Fallback: inline import ─────────────────────────────────────────
|
||||
# 7) Enqueue Celery task
|
||||
from plugins.utility.tasks import import_text_data
|
||||
import_text_data.delay(final_path, "zip", batch.id)
|
||||
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
z.extractall(tmpdir)
|
||||
|
||||
# load plants.csv
|
||||
plant_path = os.path.join(tmpdir, "plants.csv")
|
||||
with open(plant_path, newline="", encoding="utf-8-sig") as pf:
|
||||
reader = csv.DictReader(pf)
|
||||
if reader.fieldnames != PLANT_HEADERS:
|
||||
missing = set(PLANT_HEADERS) - set(reader.fieldnames or [])
|
||||
extra = set(reader.fieldnames or []) - set(PLANT_HEADERS)
|
||||
os.remove(tmp_zip.name)
|
||||
flash(f"plants.csv header mismatch. Missing: {missing}, Extra: {extra}", "danger")
|
||||
return redirect(request.url)
|
||||
plant_rows = list(reader)
|
||||
|
||||
# load media.csv
|
||||
media_path = os.path.join(tmpdir, "media.csv")
|
||||
with open(media_path, newline="", encoding="utf-8-sig") as mf:
|
||||
mreader = csv.DictReader(mf)
|
||||
if mreader.fieldnames != MEDIA_HEADERS:
|
||||
missing = set(MEDIA_HEADERS) - set(mreader.fieldnames or [])
|
||||
extra = set(mreader.fieldnames or []) - set(MEDIA_HEADERS)
|
||||
os.remove(tmp_zip.name)
|
||||
flash(f"media.csv header mismatch. Missing: {missing}, Extra: {extra}", "danger")
|
||||
return redirect(request.url)
|
||||
media_rows = list(mreader)
|
||||
|
||||
# import plants
|
||||
neo = get_neo4j_handler()
|
||||
plant_map = {}
|
||||
added_plants = 0
|
||||
|
||||
for row in plant_rows:
|
||||
# common name
|
||||
common = PlantCommonName.query.filter_by(name=row["Name"]).first()
|
||||
if not common:
|
||||
common = PlantCommonName(name=row["Name"])
|
||||
db.session.add(common)
|
||||
db.session.flush()
|
||||
|
||||
# scientific name
|
||||
scientific = PlantScientificName.query.filter_by(name=row["Scientific Name"]).first()
|
||||
if not scientific:
|
||||
scientific = PlantScientificName(
|
||||
name = row["Scientific Name"],
|
||||
common_id = common.id
|
||||
)
|
||||
db.session.add(scientific)
|
||||
db.session.flush()
|
||||
|
||||
raw_mu = row.get("Mother UUID") or None
|
||||
mu_for_insert= raw_mu if raw_mu in plant_map else None
|
||||
|
||||
p = Plant(
|
||||
uuid = row["UUID"],
|
||||
common_id = common.id,
|
||||
scientific_id = scientific.id,
|
||||
plant_type = row["Type"],
|
||||
owner_id = current_user.id,
|
||||
vendor_name = row["Vendor Name"] or None,
|
||||
price = float(row["Price"]) if row["Price"] else None,
|
||||
mother_uuid = mu_for_insert,
|
||||
notes = row["Notes"] or None,
|
||||
short_id = row.get("Short ID") or None,
|
||||
data_verified = True
|
||||
)
|
||||
db.session.add(p)
|
||||
db.session.flush()
|
||||
|
||||
plant_map[p.uuid] = p.id
|
||||
|
||||
log = PlantOwnershipLog(
|
||||
plant_id = p.id,
|
||||
user_id = current_user.id,
|
||||
date_acquired = datetime.utcnow(),
|
||||
transferred = False,
|
||||
is_verified = True
|
||||
)
|
||||
db.session.add(log)
|
||||
|
||||
neo.create_plant_node(p.uuid, row["Name"])
|
||||
if raw_mu:
|
||||
neo.create_lineage(child_uuid=p.uuid, parent_uuid=raw_mu)
|
||||
|
||||
added_plants += 1
|
||||
|
||||
db.session.commit()
|
||||
|
||||
# backfill mothers
|
||||
for row in plant_rows:
|
||||
if row.get("Mother UUID"):
|
||||
Plant.query.filter_by(uuid=row["UUID"]).update({
|
||||
'mother_uuid': row["Mother UUID"]
|
||||
})
|
||||
db.session.commit()
|
||||
|
||||
# import media images
|
||||
added_media = 0
|
||||
for mrow in media_rows:
|
||||
puuid = mrow["Plant UUID"]
|
||||
pid = plant_map.get(puuid)
|
||||
if not pid:
|
||||
continue
|
||||
|
||||
subpath = mrow["Image Path"].split('uploads/', 1)[-1]
|
||||
src = os.path.join(tmpdir, "images", subpath)
|
||||
if not os.path.isfile(src):
|
||||
continue
|
||||
|
||||
try:
|
||||
# build FileStorage for convenience
|
||||
with open(src, "rb") as f:
|
||||
fs = FileStorage(
|
||||
stream = io.BytesIO(f.read()),
|
||||
filename = os.path.basename(subpath),
|
||||
content_type='image/jpeg'
|
||||
)
|
||||
|
||||
# now save to our UPLOAD_FOLDER
|
||||
now = datetime.utcnow()
|
||||
secure_name = secure_filename(fs.filename)
|
||||
storage_dir = os.path.join(
|
||||
current_app.config["UPLOAD_FOLDER"],
|
||||
str(current_user.id),
|
||||
now.strftime("%Y/%m/%d")
|
||||
)
|
||||
os.makedirs(storage_dir, exist_ok=True)
|
||||
|
||||
unique_name = f"{uuid.uuid4().hex}_{secure_name}"
|
||||
full_path = os.path.join(storage_dir, unique_name)
|
||||
fs.save(full_path)
|
||||
|
||||
file_url = f"/{current_user.id}/{now.strftime('%Y/%m/%d')}/{unique_name}"
|
||||
|
||||
media = Media(
|
||||
plugin = "plant",
|
||||
related_id = pid,
|
||||
filename = unique_name,
|
||||
uploaded_at = datetime.fromisoformat(mrow["Uploaded At"]),
|
||||
uploader_id = current_user.id,
|
||||
caption = mrow["Source Type"],
|
||||
plant_id = pid,
|
||||
created_at = datetime.fromisoformat(mrow["Uploaded At"]),
|
||||
file_url = file_url
|
||||
)
|
||||
db.session.add(media)
|
||||
added_media += 1
|
||||
|
||||
except Exception as e:
|
||||
current_app.logger.warning(f"Failed to import media file: {subpath} → {e}")
|
||||
current_app.logger.debug(traceback.format_exc())
|
||||
|
||||
db.session.commit()
|
||||
neo.close()
|
||||
os.remove(tmp_zip.name)
|
||||
|
||||
flash(f"Imported {added_plants} plants and {added_media} images.", "success")
|
||||
flash("ZIP received; import queued in background.", "success")
|
||||
return redirect(request.url)
|
||||
|
||||
# ── CSV Review Flow ─────────────────────────────────────────────────
|
||||
@ -345,12 +191,12 @@ def upload():
|
||||
suggested = all_sci[suggestions[0]].name
|
||||
|
||||
item = {
|
||||
"uuid": uuid_val,
|
||||
"name": name,
|
||||
"sci_name": sci_name,
|
||||
"suggested": suggested,
|
||||
"plant_type": plant_type,
|
||||
"mother_uuid": mother_uuid
|
||||
"uuid" : uuid_val,
|
||||
"name" : name,
|
||||
"sci_name" : sci_name,
|
||||
"suggested" : suggested,
|
||||
"plant_type" : plant_type,
|
||||
"mother_uuid" : mother_uuid
|
||||
}
|
||||
review_list.append(item)
|
||||
session["pending_rows"].append(item)
|
||||
@ -399,6 +245,9 @@ def upload():
|
||||
flash("File uploaded and saved successfully.", "success")
|
||||
return redirect(request.url)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# GET → render form
|
||||
# -------------------------------------------------------------------------
|
||||
return render_template("utility/upload.html", csrf_token=generate_csrf())
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user