Skip to content

Commit c23522b

Browse files
AlvaroMarquesAndraderalphrass
authored andcommitted
[MLOP-636] Create migration classes (#282)
1 parent 6b9ec18 commit c23522b

File tree

3 files changed

+108
-0
lines changed

3 files changed

+108
-0
lines changed
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
"""Cassandra Migration entity."""
2+
3+
from typing import Any, Dict, List
4+
5+
from butterfree.migrations import DatabaseMigration
6+
7+
8+
class CassandraMigration(DatabaseMigration):
9+
"""Cassandra class for Migrations."""
10+
11+
def create_query(
12+
self,
13+
fs_schema: List[Dict[str, Any]],
14+
db_schema: List[Dict[str, Any]],
15+
table_name: str,
16+
) -> Any:
17+
"""Create a query regarding Cassandra.
18+
19+
Returns:
20+
Schema object.
21+
22+
"""
23+
pass
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
"""Metastore Migration entity."""
2+
3+
from typing import Any, Dict, List
4+
5+
from butterfree.migrations import DatabaseMigration
6+
7+
8+
class MetastoreMigration(DatabaseMigration):
9+
"""Metastore class for Migrations."""
10+
11+
def create_query(
12+
self,
13+
fs_schema: List[Dict[str, Any]],
14+
db_schema: List[Dict[str, Any]],
15+
table_name: str,
16+
) -> Any:
17+
"""Create a query regarding Metastore.
18+
19+
Returns:
20+
Schema object.
21+
22+
"""
23+
pass

butterfree/migrations/migration.py

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
"""Migration entity."""
2+
3+
from abc import ABC, abstractmethod
4+
from typing import Any, Callable, Dict, List
5+
6+
from butterfree.pipelines import FeatureSetPipeline
7+
8+
9+
class DatabaseMigration(ABC):
10+
"""Abstract base class for Migrations."""
11+
12+
@abstractmethod
13+
def create_query(
14+
self,
15+
fs_schema: List[Dict[str, Any]],
16+
db_schema: List[Dict[str, Any]],
17+
table_name: str,
18+
) -> Any:
19+
"""Create a query regarding a data source.
20+
21+
Returns:
22+
The desired query for the given database.
23+
24+
"""
25+
26+
def _validate_schema(
27+
self, fs_schema: List[Dict[str, Any]], db_schema: List[Dict[str, Any]]
28+
) -> Any:
29+
"""Provides schema validation for feature sets.
30+
31+
Compares the schema of your local feature set to the
32+
corresponding table in a given database.
33+
34+
Args:
35+
fs_schema: object that contains feature set's schemas.
36+
db_schema: object that contains the table og a given db schema.
37+
38+
"""
39+
40+
def _get_schema(self, db_client: Callable, table_name: str) -> List[Dict[str, Any]]:
41+
"""Get a table schema in the respective database.
42+
43+
Returns:
44+
Schema object.
45+
"""
46+
pass
47+
48+
def _apply_migration(self, query: str, db_client: Callable) -> None:
49+
"""Apply the migration in the respective database."""
50+
51+
def _send_logs_to_s3(self) -> None:
52+
"""Send all migration logs to S3."""
53+
pass
54+
55+
def run(self, pipelines: List[FeatureSetPipeline]) -> None:
56+
"""Runs the migrations.
57+
58+
Args:
59+
pipelines: the feature set pipelines.
60+
61+
"""
62+
pass

0 commit comments

Comments
 (0)