Implemented alembic migrations, added cursors closing task.

data/app_types.py

@@ -0,0 +1,63 @@
+import datetime
+from aiomysql import SSCursor, Connection, Pool, SSCursor
+from data.schemas import SelectQuery
+from core.exceptions import ClosedCursorUsage
+
+
+class CachedCursor:
+    def __init__(
+        self,
+        id: str,
+        cursor: SSCursor,
+        connection: Connection,
+        pool: Pool,
+        connection_id: int,
+        query: SelectQuery,
+        ttl: int=60
+    ):
+
+        self.id = id
+        self.cursor = cursor
+        self.connection = connection
+        self.connection_id = connection_id
+        self.pool = pool
+        self.query = query
+        self.row_count: int = -1 if cursor.rowcount > 10000000000000000000 else cursor.rowcount
+        # The rowcount for a SELECT is set to -1 when using a server-side cursor.
+        # The incorrect large number (> 10000000000000000000) is because -1 is
+        #  interpreted as an unsigned integer in MySQL's internal C API.
+        self.fetched_rows: int = 0
+        self.is_closed: bool=False
+        self.ttl:int = ttl
+        self.close_at = self.upgrade_close_at()
+        self.done=False
+
+    @property
+    def has_more(self):
+        return not self.done
+    
+    def upgrade_close_at(self) -> int:
+        return int(datetime.datetime.now(tz=datetime.UTC).timestamp()) + self.ttl
+    
+    async def close(self):
+        await self.cursor.close()
+        await self.pool.release(self.connection)
+        self.is_closed=True
+
+    async def fetch_many(self, size: int = 100) -> tuple[list[tuple], bool]:
+        if self.is_closed:
+            raise ClosedCursorUsage
+        
+        result = await self.cursor.fetchmany(size)
+
+        if len(result) < size:
+            # The cursor has reached the end of the set.
+            await self.close()
+            self.done=True
+        else:
+            self.upgrade_close_at()
+
+        self.fetched_rows += len(result)
+        return result
+
+

data/models.py

@@ -21,9 +21,11 @@ class Connection(Base):
     db_name = Column(String, nullable=False)
     type = Column(Enum(ConnectionTypes), nullable=False)
     host = Column(String)
-    port = Column(Integer)
+    port = Column(Integer)    
     username = Column(String)
     password = Column(String)
+    pool_minsize = Column(Integer, nullable=False, default=5)
+    pool_maxsize = Column(Integer, nullable=False, default=10)
     owner_id = Column(Integer, ForeignKey("users.id"))
 
     # owner = relationship("User", back_populates="connections")

data/schemas.py

@@ -124,8 +124,8 @@ class SelectQueryBase(BaseModel):
     columns: Union[Literal["*"], List[str]] = "*"
     filters: Optional[List[FilterClause]] = None
     sort_by: Optional[List[SortClause]] = None
-    limit: Annotated[int, Field(strict=True, gt=0)] = None
-    offset: Annotated[int, Field(strict=True, ge=0)] = None
+    limit: Optional[Annotated[int, Field(strict=True, gt=0)]] = None
+    offset: Optional[Annotated[int, Field(strict=True, ge=0)]] = None
 
     @field_validator("table_name")
     @classmethod
@@ -179,16 +179,20 @@ class SelectQueryInResult(BaseModel):
         from_attributes = True
 
 
-class SelectQueryMetaData(BaseModel):
-    cursor: Optional[UUID4] = Field(
-        None,
-        description="A UUID4 cursor for pagination. Can be None if no more data is available.",
-    )
-    total_number: int
-    has_more: bool = False
+class CachedCursorOut(BaseModel):
+    id: UUID4 | None
+    connection_id: int
+    query: SelectQueryInResult
+    row_count: int
+    fetched_rows: int
+    is_closed: bool
+    has_more: bool
+    close_at: int
+    ttl: int
+    class Config:
+        from_attributes = True
 
 
 class SelectResult(BaseModel):
-    meta: SelectQueryMetaData
-    query: SelectQueryInResult
+    cursor: CachedCursorOut
     results: SelectResultData | None