From 85cf2157db1733935786dcad253ea457462d7a0d Mon Sep 17 00:00:00 2001
From: Colin Goutte <cgte@bk.ru>
Date: Sun, 27 Aug 2023 14:35:02 +0200
Subject: [PATCH] Simple pagination creates great speed impovement for import

---
 Makefile    | 2 +-
 crud.py     | 6 ++++--
 database.py | 9 +++++++--
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 9a2e9c5..403e1e2 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ refactor_tdd:
 
 
 watch_db:
-	watch "sqlite3  sql_app.db  'select * from movies'"
+	watch "sqlite3  sql_app.db  'select count(*) from movies'"
 
 
 test:
diff --git a/crud.py b/crud.py
index 7af3d0a..bdaa751 100644
--- a/crud.py
+++ b/crud.py
@@ -7,6 +7,7 @@ import models
 
 def create_movie(
     db: Session,
+    batch_mode=False,
     *,
     title: str,
     genres: list[str],
@@ -25,8 +26,9 @@ def create_movie(
         release_date=release_date,
     )
     db.add(db_movie)
-    db.commit()
-    db.refresh(db_movie)
+    if not batch_mode:
+        db.commit()
+        db.refresh(db_movie)
     return db_movie
 
 
diff --git a/database.py b/database.py
index 5727b1e..f3a39ee 100644
--- a/database.py
+++ b/database.py
@@ -44,10 +44,15 @@ def fill_db(
     import crud
     import csv
 
+    page_size = 1_00
     with open(movie_input_file) as csvfile:
-        for movie_data in csv.DictReader(csvfile):
+        for count, movie_data in enumerate(csv.DictReader(csvfile), start=1):
             adapted_data = adapt_movie_data(movie_data)
-            crud.create_movie(db, **adapted_data)
+            crud.create_movie(db, batch_mode=True, **adapted_data)
+            if count % page_size == 0:
+                db.commit()
+
+        db.commit()
 
 
 if __name__ == "__main__":