:fix: Delete created files on failed import

This commit is contained in:
itskovacs 2025-11-11 18:23:06 +01:00
parent cfe4baa794
commit e8c9fe86f2

View File

@ -19,7 +19,7 @@ from ..models.models import (Backup, BackupStatus, Category, CategoryRead,
TripRead, User, UserRead) TripRead, User, UserRead)
from .date import dt_utc, iso_to_dt from .date import dt_utc, iso_to_dt
from .utils import (assets_folder_path, attachments_trip_folder_path, from .utils import (assets_folder_path, attachments_trip_folder_path,
b64img_decode, save_image_to_file) b64img_decode, remove_image, save_image_to_file)
def process_backup_export(session: SessionDep, backup_id: int): def process_backup_export(session: SessionDep, backup_id: int):
@ -147,79 +147,47 @@ async def process_backup_import(
except Exception: except Exception:
raise HTTPException(status_code=400, detail="Invalid file") raise HTTPException(status_code=400, detail="Invalid file")
try: with ZipFile(io.BytesIO(zip_content), "r") as zipf:
with ZipFile(io.BytesIO(zip_content), "r") as zipf: zip_filenames = zipf.namelist()
zip_filenames = zipf.namelist() if "data.json" not in zip_filenames:
if "data.json" not in zip_filenames: raise HTTPException(status_code=400, detail="Invalid file")
raise HTTPException(status_code=400, detail="Invalid file")
try: try:
data = json.loads(zipf.read("data.json")) data = json.loads(zipf.read("data.json"))
except Exception: except Exception:
raise HTTPException(status_code=400, detail="Invalid file") raise HTTPException(status_code=400, detail="Invalid file")
image_files = { image_files = {
path.split("/")[-1]: path path.split("/")[-1]: path
for path in zip_filenames for path in zip_filenames
if path.startswith("images/") and not path.endswith("/") if path.startswith("images/") and not path.endswith("/")
}
attachment_files = {
path.split("/")[-1]: path
for path in zip_filenames
if path.startswith("attachments/") and not path.endswith("/")
}
error_details = "Bad request"
created_image_filenames = []
created_attachment_trips = []
try:
existing_categories = {
category.name: category
for category in session.exec(select(Category).where(Category.user == current_user)).all()
} }
attachment_files = { categories_to_add = []
path.split("/")[-1]: path for category in data.get("categories", []):
for path in zip_filenames category_name = category.get("name")
if path.startswith("attachments/") and not path.endswith("/") category_exists = existing_categories.get(category_name)
}
try: if category_exists:
existing_categories = { if category.get("color"):
category.name: category category_exists.color = category["color"]
for category in session.exec(select(Category).where(Category.user == current_user)).all()
}
categories_to_add = [] if category.get("image_id") and category.get("image_id") != category_exists.image_id:
for category in data.get("categories", []):
category_name = category.get("name")
category_exists = existing_categories.get(category_name)
if category_exists:
if category.get("color"):
category_exists.color = category["color"]
if category.get("image_id") and category.get("image_id") != category_exists.image_id:
category_filename = category.get("image").split("/")[-1]
if category_filename and category_filename in image_files:
try:
image_bytes = zipf.read(image_files[category_filename])
filename = save_image_to_file(image_bytes, settings.PLACE_IMAGE_SIZE)
if filename:
image = Image(filename=filename, user=current_user)
session.add(image)
session.flush()
session.refresh(image)
if category_exists.image_id:
old_image = session.get(Image, category_exists.image_id)
if old_image:
session.delete(old_image)
category_exists.image_id = None
session.flush()
category_exists.image_id = image.id
except Exception:
pass
session.add(category_exists)
existing_categories[category_name] = category_exists
continue
new_category = {
key: category[key]
for key in category.keys()
if key not in {"id", "image", "image_id"}
}
new_category["user"] = current_user
if category.get("image_id"):
category_filename = category.get("image").split("/")[-1] category_filename = category.get("image").split("/")[-1]
if category_filename and category_filename in image_files: if category_filename and category_filename in image_files:
try: try:
@ -230,266 +198,317 @@ async def process_backup_import(
session.add(image) session.add(image)
session.flush() session.flush()
session.refresh(image) session.refresh(image)
new_category["image_id"] = image.id created_image_filenames.append(filename)
if category_exists.image_id:
old_image = session.get(Image, category_exists.image_id)
if old_image:
session.delete(old_image)
category_exists.image_id = None
session.flush()
category_exists.image_id = image.id
except Exception: except Exception:
pass pass
new_category = Category(**new_category) session.add(category_exists)
categories_to_add.append(new_category) existing_categories[category_name] = category_exists
session.add(new_category) continue
if categories_to_add: new_category = {
session.flush() key: category[key] for key in category.keys() if key not in {"id", "image", "image_id"}
for category in categories_to_add: }
existing_categories[category.name] = category new_category["user"] = current_user
places = [] if category.get("image_id"):
places_to_add = [] category_filename = category.get("image").split("/")[-1]
for place in data.get("places", []): if category_filename and category_filename in image_files:
category_name = place.get("category", {}).get("name") try:
category = existing_categories.get(category_name) image_bytes = zipf.read(image_files[category_filename])
if not category: filename = save_image_to_file(image_bytes, settings.PLACE_IMAGE_SIZE)
if filename:
image = Image(filename=filename, user=current_user)
session.add(image)
session.flush()
session.refresh(image)
created_image_filenames.append(filename)
new_category["image_id"] = image.id
except Exception:
pass
new_category = Category(**new_category)
categories_to_add.append(new_category)
session.add(new_category)
if categories_to_add:
session.flush()
for category in categories_to_add:
existing_categories[category.name] = category
places = []
places_to_add = []
for place in data.get("places", []):
category_name = place.get("category", {}).get("name")
category = existing_categories.get(category_name)
if not category:
continue
new_place = {
key: place[key]
for key in place.keys()
if key not in {"id", "image", "image_id", "category", "category_id"}
}
new_place["user"] = current_user
new_place["category_id"] = category.id
if place.get("image_id"):
place_filename = place.get("image").split("/")[-1]
if place_filename and place_filename in image_files:
try:
image_bytes = zipf.read(image_files[place_filename])
filename = save_image_to_file(image_bytes, settings.PLACE_IMAGE_SIZE)
if filename:
image = Image(filename=filename, user=current_user)
session.add(image)
session.flush()
session.refresh(image)
created_image_filenames.append(filename)
new_place["image_id"] = image.id
except Exception:
pass
new_place = Place(**new_place)
places_to_add.append(new_place)
places.append(new_place)
if places_to_add:
session.add_all(places_to_add)
session.flush()
db_user = session.get(User, current_user)
if data.get("settings") and db_user:
settings_data = data["settings"]
setting_fields = [
"map_lat",
"map_lng",
"currency",
"tile_layer",
"mode_low_network",
"mode_dark",
"mode_gpx_in_place",
]
for field in setting_fields:
if field in settings_data:
setattr(db_user, field, settings_data[field])
if "do_not_display" in settings_data:
db_user.do_not_display = ",".join(settings_data["do_not_display"])
session.add(db_user)
session.flush()
trip_place_id_map = {p["id"]: new_p.id for p, new_p in zip(data.get("places", []), places)}
items_to_add = []
packing_to_add = []
checklist_to_add = []
attachment_links_to_add = []
for trip in data.get("trips", []):
new_trip = {
key: trip[key]
for key in trip.keys()
if key
not in {
"id",
"image",
"image_id",
"places",
"days",
"shared",
"collaborators",
"attachments",
"packing_items",
"checklist_items",
}
}
new_trip["user"] = current_user
if trip.get("image_id"):
trip_filename = trip.get("image").split("/")[-1]
if trip_filename and trip_filename in image_files:
try:
image_bytes = zipf.read(image_files[trip_filename])
filename = save_image_to_file(image_bytes, settings.TRIP_IMAGE_SIZE)
if filename:
image = Image(filename=filename, user=current_user)
session.add(image)
session.flush()
session.refresh(image)
created_image_filenames.append(filename)
new_trip["image_id"] = image.id
except Exception:
pass
new_trip = Trip(**new_trip)
session.add(new_trip)
session.flush()
session.refresh(new_trip)
for place in trip.get("places", []):
old_id = place.get("id")
new_place_id = trip_place_id_map.get(old_id)
if new_place_id:
db_place = session.get(Place, new_place_id)
if db_place:
new_trip.places.append(db_place)
trip_attachment_mapping = {}
for attachment in trip.get("attachments", []):
stored_filename = attachment.get("stored_filename")
old_attachment_id = attachment.get("id")
if not stored_filename or not old_attachment_id:
continue continue
new_place = { if stored_filename in attachment_files:
key: place[key] try:
for key in place.keys() attachment_bytes = zipf.read(attachment_files[stored_filename])
if key not in {"id", "image", "image_id", "category", "category_id"} new_attachment = {
} key: attachment[key]
new_place["user"] = current_user for key in attachment
new_place["category_id"] = category.id if key not in {"id", "trip_id", "trip", "uploaded_by"}
}
new_attachment["trip_id"] = new_trip.id
new_attachment["uploaded_by"] = current_user
new_attachment_obj = TripAttachment(**new_attachment)
if place.get("image_id"): attachment_path = attachments_trip_folder_path(new_trip.id) / stored_filename
place_filename = place.get("image").split("/")[-1] created_attachment_trips.append(new_trip.id)
if place_filename and place_filename in image_files: attachment_path.write_bytes(attachment_bytes)
try: session.add(new_attachment_obj)
image_bytes = zipf.read(image_files[place_filename]) session.flush()
filename = save_image_to_file(image_bytes, settings.PLACE_IMAGE_SIZE) session.refresh(new_attachment_obj)
if filename: trip_attachment_mapping[old_attachment_id] = new_attachment_obj.id
image = Image(filename=filename, user=current_user)
session.add(image)
session.flush()
session.refresh(image)
new_place["image_id"] = image.id
except Exception:
pass
new_place = Place(**new_place) except Exception:
places_to_add.append(new_place)
places.append(new_place)
if places_to_add:
session.add_all(places_to_add)
session.flush()
db_user = session.get(User, current_user)
if data.get("settings") and db_user:
settings_data = data["settings"]
setting_fields = [
"map_lat",
"map_lng",
"currency",
"tile_layer",
"mode_low_network",
"mode_dark",
"mode_gpx_in_place",
]
for field in setting_fields:
if field in settings_data:
setattr(db_user, field, settings_data[field])
if "do_not_display" in settings_data:
db_user.do_not_display = ",".join(settings_data["do_not_display"])
session.add(db_user)
session.flush()
trip_place_id_map = {p["id"]: new_p.id for p, new_p in zip(data.get("places", []), places)}
items_to_add = []
packing_to_add = []
checklist_to_add = []
attachment_links_to_add = []
for trip in data.get("trips", []):
new_trip = {
key: trip[key]
for key in trip.keys()
if key
not in {
"id",
"image",
"image_id",
"places",
"days",
"shared",
"collaborators",
"attachments",
"packing_items",
"checklist_items",
}
}
new_trip["user"] = current_user
if trip.get("image_id"):
trip_filename = trip.get("image").split("/")[-1]
if trip_filename and trip_filename in image_files:
try:
image_bytes = zipf.read(image_files[trip_filename])
filename = save_image_to_file(image_bytes, settings.TRIP_IMAGE_SIZE)
if filename:
image = Image(filename=filename, user=current_user)
session.add(image)
session.flush()
session.refresh(image)
new_trip["image_id"] = image.id
except Exception:
pass
new_trip = Trip(**new_trip)
session.add(new_trip)
session.flush()
session.refresh(new_trip)
for place in trip.get("places", []):
old_id = place.get("id")
new_place_id = trip_place_id_map.get(old_id)
if new_place_id:
db_place = session.get(Place, new_place_id)
if db_place:
new_trip.places.append(db_place)
trip_attachment_mapping = {}
for attachment in trip.get("attachments", []):
stored_filename = attachment.get("stored_filename")
old_attachment_id = attachment.get("id")
if not stored_filename or not old_attachment_id:
continue continue
if stored_filename in attachment_files: for day in trip.get("days", []):
try: day_data = {key: day[key] for key in day if key not in {"id", "items"}}
attachment_bytes = zipf.read(attachment_files[stored_filename]) if "dt" in day_data and isinstance(day_data["dt"], str):
new_attachment = { day_data["dt"] = iso_to_dt(day_data["dt"])
key: attachment[key] new_day = TripDay(**day_data, trip_id=new_trip.id)
for key in attachment session.add(new_day)
if key not in {"id", "trip_id", "trip"} session.flush()
}
new_attachment["trip_id"] = new_trip.id
new_attachment["user"] = current_user
new_attachment_obj = TripAttachment(**new_attachment)
attachment_path = attachments_trip_folder_path(new_trip.id) / stored_filename for item in day.get("items", []):
attachment_path.write_bytes(attachment_bytes) if item.get("paid_by"):
session.add(new_attachment_obj) u = item.get("paid_by")
session.flush() db_user = session.get(User, u)
session.refresh(new_attachment_obj) if not db_user:
trip_attachment_mapping[old_attachment_id] = new_attachment_obj.id error_details = f"User <{u}> does not exist and is specified in Paid By"
raise
except Exception: item_data = {
continue key: item[key]
for key in item
if key not in {"id", "place", "place_id", "image", "image_id", "attachments"}
}
item_data["day_id"] = new_day.id
for day in trip.get("days", []): place = item.get("place")
day_data = {key: day[key] for key in day if key not in {"id", "items"}} if place and (place_id := place.get("id")):
if "dt" in day_data and isinstance(day_data["dt"], str): new_place_id = trip_place_id_map.get(place_id)
day_data["dt"] = iso_to_dt(day_data["dt"]) item_data["place_id"] = new_place_id
new_day = TripDay(**day_data, trip_id=new_trip.id)
session.add(new_day) if item_data.get("image_id"):
place_filename = place.get("image").split("/")[-1]
if place_filename and place_filename in image_files:
try:
image_bytes = zipf.read(image_files[place_filename])
filename = save_image_to_file(image_bytes, settings.PLACE_IMAGE_SIZE)
if filename:
image = Image(filename=filename, user=current_user)
session.add(image)
session.flush()
session.refresh(image)
created_image_filenames.append(filename)
item_data["image_id"] = image.id
except Exception:
pass
trip_item = TripItem(**item_data)
session.add(trip_item)
session.flush() session.flush()
session.refresh(trip_item)
items_to_add.append(trip_item)
for item in day.get("items", []): for attachment in item.get("attachments", []):
item_data = { attachment_id = attachment.get("id")
key: item[key] if attachment_id and attachment_id in trip_attachment_mapping:
for key in item new_attachment_id = trip_attachment_mapping[attachment_id]
if key not in {"id", "place", "place_id", "image", "image_id", "attachments"} link = TripItemAttachmentLink(
} item_id=trip_item.id, attachment_id=new_attachment_id
item_data["day_id"] = new_day.id )
item_data["user"] = current_user attachment_links_to_add.append(link)
place = item.get("place") for item in trip.get("packing_items", []):
if place and (place_id := place.get("id")): new_packing = {
new_place_id = trip_place_id_map.get(place_id) key: item[key] for key in item.keys() if key not in {"id", "trip_id", "trip"}
item_data["place_id"] = new_place_id }
new_packing["trip_id"] = new_trip.id
packing_to_add.append(TripPackingListItem(**new_packing))
if item_data.get("image_id"): for item in trip.get("checklist_items", []):
place_filename = place.get("image").split("/")[-1] new_checklist = {
if place_filename and place_filename in image_files: key: item[key] for key in item.keys() if key not in {"id", "trip_id", "trip"}
try: }
image_bytes = zipf.read(image_files[place_filename]) new_checklist["trip_id"] = new_trip.id
filename = save_image_to_file(image_bytes, settings.PLACE_IMAGE_SIZE) checklist_to_add.append(TripChecklistItem(**new_checklist))
if filename:
image = Image(filename=filename, user=current_user)
session.add(image)
session.flush()
session.refresh(image)
item_data["image_id"] = image.id
except Exception:
pass
trip_item = TripItem(**item_data) if attachment_links_to_add:
session.add(trip_item) session.add_all(attachment_links_to_add)
session.flush()
session.refresh(trip_item)
items_to_add.append(trip_item)
for attachment in item.get("attachments", []): if items_to_add:
attachment_id = attachment.get("id") session.add_all(items_to_add)
if attachment_id and attachment_id in trip_attachment_mapping:
new_attachment_id = trip_attachment_mapping[attachment_id]
link = TripItemAttachmentLink(
item_id=trip_item.id, attachment_id=new_attachment_id
)
attachment_links_to_add.append(link)
for item in trip.get("packing_items", []): if packing_to_add:
new_packing = { session.add_all(packing_to_add)
key: item[key] for key in item.keys() if key not in {"id", "trip_id", "trip"}
}
new_packing["trip_id"] = new_trip.id
packing_to_add.append(TripPackingListItem(**new_packing))
for item in trip.get("checklist_items", []): if checklist_to_add:
new_checklist = { session.add_all(checklist_to_add)
key: item[key] for key in item.keys() if key not in {"id", "trip_id", "trip"}
}
new_checklist["trip_id"] = new_trip.id
checklist_to_add.append(TripChecklistItem(**new_checklist))
if attachment_links_to_add: # BOOM!
session.add_all(attachment_links_to_add) session.commit()
if items_to_add: return {
session.add_all(items_to_add) "places": [PlaceRead.serialize(p) for p in places],
"categories": [
CategoryRead.serialize(c)
for c in session.exec(
select(Category)
.options(selectinload(Category.image))
.where(Category.user == current_user)
).all()
],
"settings": UserRead.serialize(session.get(User, current_user)),
}
if packing_to_add: except Exception:
session.add_all(packing_to_add) session.rollback()
for filename in created_image_filenames:
if checklist_to_add: remove_image(filename)
session.add_all(checklist_to_add) for trip_id in created_attachment_trips:
try:
# BOOM! folder = attachments_trip_folder_path(trip_id)
session.commit() if not folder.exists():
return
return { for file in folder.iterdir():
"places": [PlaceRead.serialize(p) for p in places], file.unlink()
"categories": [ folder.rmdir()
CategoryRead.serialize(c) except Exception:
for c in session.exec( pass
select(Category) raise HTTPException(status_code=400, detail=error_details)
.options(selectinload(Category.image))
.where(Category.user == current_user)
).all()
],
"settings": UserRead.serialize(session.get(User, current_user)),
}
except Exception as exc:
session.rollback()
print(exc)
raise HTTPException(status_code=400, detail="Bad request")
except Exception as exc:
print(exc)
raise HTTPException(status_code=400, detail="Bad request")
async def process_legacy_import( async def process_legacy_import(