Simple pagination creates great speed impovement for import

Makefile

@@ -22,7 +22,7 @@ refactor_tdd:
 
 
 watch_db:
-	watch "sqlite3  sql_app.db  'select * from movies'"
+	watch "sqlite3  sql_app.db  'select count(*) from movies'"
 
 
 test:

crud.py

@@ -7,6 +7,7 @@ import models
 
 def create_movie(
     db: Session,
+    batch_mode=False,
     *,
     title: str,
     genres: list[str],
@@ -25,8 +26,9 @@ def create_movie(
         release_date=release_date,
     )
     db.add(db_movie)
-    db.commit()
-    db.refresh(db_movie)
+    if not batch_mode:
+        db.commit()
+        db.refresh(db_movie)
     return db_movie
 
 

database.py

@@ -44,10 +44,15 @@ def fill_db(
     import crud
     import csv
 
+    page_size = 1_00
     with open(movie_input_file) as csvfile:
-        for movie_data in csv.DictReader(csvfile):
+        for count, movie_data in enumerate(csv.DictReader(csvfile), start=1):
             adapted_data = adapt_movie_data(movie_data)
-            crud.create_movie(db, **adapted_data)
+            crud.create_movie(db, batch_mode=True, **adapted_data)
+            if count % page_size == 0:
+                db.commit()
+
+        db.commit()
 
 
 if __name__ == "__main__":