This commit is contained in:
Christian Mantha
2026-03-02 19:10:52 -05:00
commit 2ca0b9ef7c
28907 changed files with 5233713 additions and 0 deletions

View File

@@ -0,0 +1,74 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = mlflow/store/db_migrations
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# timezone to use when rendering the date
# within the migration file as well as the filename.
# string value is passed to dateutil.tz.gettz()
# leave blank for localtime
# timezone =
# max length of characters to apply to the
# "slug" field
# truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# version location specification; this defaults
# to alembic/versions. When using multiple version
# directories, initial revisions must be specified with --version-path
# version_locations = %(here)s/bar %(here)s/bat alembic/versions
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
sqlalchemy.url = ""
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

View File

@@ -0,0 +1,84 @@
from logging.config import fileConfig
from alembic import context
from sqlalchemy import engine_from_config, pool
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name, disable_existing_loggers=False)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
from mlflow.store.db.base_sql_model import Base
target_metadata = Base.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
# Try https://stackoverflow.com/questions/30378233/sqlite-lack-of-alter-support-alembic-migration-failing-because-of-this-solutio
context.configure(
url=url, target_metadata=target_metadata, literal_binds=True, render_as_batch=True
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# If available, use a shared connection for the database upgrade, ensuring that any
# connection-dependent state (e.g., the state of an in-memory database) is preserved
# for reference by the upgrade routine. For more information, see
# https://alembic.sqlalchemy.org/en/latest/cookbook.html#sharing-a-
# connection-with-a-series-of-migration-commands-and-environments
connection = config.attributes.get("connection")
if connection is None:
engine = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
else:
engine = connection.engine
with engine.connect() as connection:
context.configure(
connection=connection, target_metadata=target_metadata, render_as_batch=True
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

View File

@@ -0,0 +1,88 @@
"""add cascading deletion to datasets from experiments
Revision ID: 0584bdc529eb
Revises: f5a4f2784254
Create Date: 2024-11-11 15:27:53.189685
"""
import sqlalchemy as sa
from alembic import op
from mlflow.exceptions import MlflowException
from mlflow.store.tracking.dbmodels.models import SqlDataset, SqlExperiment
# revision identifiers, used by Alembic.
revision = "0584bdc529eb"
down_revision = "f5a4f2784254"
branch_labels = None
depends_on = None
def get_datasets_experiment_fk_name():
conn = op.get_bind()
metadata = sa.MetaData()
metadata.bind = conn
datasets_table = sa.Table(
SqlDataset.__tablename__,
metadata,
autoload_with=conn,
)
for constraint in datasets_table.foreign_key_constraints:
if (
constraint.referred_table.name == SqlExperiment.__tablename__
and constraint.column_keys[0] == "experiment_id"
):
return constraint.name
raise MlflowException(
"Unable to find the foreign key constraint name from datasets to experiments. "
"All foreign key constraints in datasets table: \n"
f"{datasets_table.foreign_key_constraints}"
)
def upgrade():
dialect_name = op.get_context().dialect.name
# standardize the constraint to sqlite naming convention
new_fk_constraint_name = (
f"fk_{SqlDataset.__tablename__}_experiment_id_{SqlExperiment.__tablename__}"
)
if dialect_name == "sqlite":
# Only way to drop unnamed fk constraint in sqllite
# See https://alembic.sqlalchemy.org/en/latest/batch.html#dropping-unnamed-or-named-foreign-key-constraints
with op.batch_alter_table(
SqlDataset.__tablename__,
schema=None,
naming_convention={
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
},
) as batch_op:
# in SQLite, constraint.name is None, so we have to hardcode it
batch_op.drop_constraint(new_fk_constraint_name, type_="foreignkey")
# Need to explicitly name the fk constraint with batch alter table
batch_op.create_foreign_key(
new_fk_constraint_name,
SqlExperiment.__tablename__,
["experiment_id"],
["experiment_id"],
ondelete="CASCADE",
)
else:
old_fk_constraint_name = get_datasets_experiment_fk_name()
op.drop_constraint(old_fk_constraint_name, SqlDataset.__tablename__, type_="foreignkey")
op.create_foreign_key(
new_fk_constraint_name,
SqlDataset.__tablename__,
SqlExperiment.__tablename__,
["experiment_id"],
["experiment_id"],
ondelete="CASCADE",
)
def downgrade():
pass

View File

@@ -0,0 +1,49 @@
"""drop_duplicate_killed_constraint
Revision ID: 0a8213491aaa
Revises: cfd24bdc0731
Create Date: 2020-01-28 15:26:14.757445
This migration drops a duplicate constraint on the `runs.status` column that was left as a byproduct
of an erroneous implementation of the `cfd24bdc0731_update_run_status_constraint_with_killed`
migration in MLflow 1.5. The implementation of this migration has since been fixed.
"""
import logging
from alembic import op
_logger = logging.getLogger(__name__)
# revision identifiers, used by Alembic.
revision = "0a8213491aaa"
down_revision = "cfd24bdc0731"
branch_labels = None
depends_on = None
def upgrade():
# Attempt to drop any existing `status` constraints on the `runs` table. This operation
# may fail against certain backends with different classes of Exception. For example,
# in MySQL <= 8.0.15, dropping constraints produces an invalid `ALTER TABLE` expression.
# Further, in certain versions of sqlite, `ALTER` (which is invoked by `drop_constraint`)
# is unsupported on `CHECK` constraints. Accordingly, we catch the generic `Exception`
# object because the failure modes are not well-enumerated or consistent across database
# backends. Because failures automatically stop batch operations and the `drop_constraint()`
# operation is expected to fail under certain circumstances, we execute `drop_constraint()`
# outside of the batch operation context.
try:
# For other database backends, the status check constraint is dropped by
# cfd24bdc0731_update_run_status_constraint_with_killed.py
if op.get_bind().engine.name == "mysql":
op.drop_constraint(constraint_name="status", table_name="runs", type_="check")
except Exception as e:
_logger.warning(
"Failed to drop check constraint. Dropping check constraints may not be supported"
" by your SQL database. Exception content: %s",
e,
)
def downgrade():
pass

View File

@@ -0,0 +1,24 @@
"""add deleted_time field to runs table
Revision ID: 0c779009ac13
Revises: bd07f7e963c5
Create Date: 2022-07-27 14:13:36.162861
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "0c779009ac13"
down_revision = "bd07f7e963c5"
branch_labels = None
depends_on = None
def upgrade():
op.add_column("runs", sa.Column("deleted_time", sa.BigInteger, nullable=True, default=None))
def downgrade():
pass

View File

@@ -0,0 +1,35 @@
"""allow nulls for metric values
Revision ID: 181f10493468
Revises: 90e64c465722
Create Date: 2019-07-10 22:40:18.787993
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "181f10493468"
down_revision = "90e64c465722"
branch_labels = None
depends_on = None
def upgrade():
with op.batch_alter_table("metrics") as batch_op:
batch_op.alter_column("value", type_=sa.types.Float(precision=53), nullable=False)
batch_op.add_column(
sa.Column(
"is_nan", sa.Boolean(create_constraint=False), nullable=False, server_default="0"
)
)
batch_op.drop_constraint(constraint_name="metric_pk", type_="primary")
batch_op.create_primary_key(
constraint_name="metric_pk",
columns=["key", "timestamp", "step", "run_uuid", "value", "is_nan"],
)
def downgrade():
pass

View File

@@ -0,0 +1,38 @@
"""add model version tags table
Revision ID: 27a6a02d2cf1
Revises: 728d730b5ebd
Create Date: 2020-06-26 13:30:27.611086
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
from mlflow.store.model_registry.dbmodels.models import SqlModelVersionTag
revision = "27a6a02d2cf1"
down_revision = "728d730b5ebd"
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
SqlModelVersionTag.__tablename__,
sa.Column("key", sa.String(length=250), primary_key=True, nullable=False),
sa.Column("value", sa.String(length=5000)),
sa.Column("name", sa.String(length=256), primary_key=True, nullable=False),
sa.Column("version", sa.Integer(), primary_key=True, nullable=False),
sa.ForeignKeyConstraint(
("name", "version"),
("model_versions.name", "model_versions.version"),
onupdate="cascade",
),
sa.PrimaryKeyConstraint("key", "name", "version", name="model_version_tag_pk"),
)
def downgrade():
pass

View File

@@ -0,0 +1,77 @@
"""add model registry tables to db
Revision ID: 2b4d017a5e9b
Revises: 89d4b8295536
Create Date: 2019-10-14 12:20:12.874424
"""
import logging
import time
from alembic import op
from sqlalchemy import (
BigInteger,
Column,
ForeignKey,
Integer,
PrimaryKeyConstraint,
String,
orm,
)
from mlflow.entities.model_registry.model_version_stages import STAGE_NONE
from mlflow.entities.model_registry.model_version_status import ModelVersionStatus
from mlflow.store.model_registry.dbmodels.models import SqlModelVersion, SqlRegisteredModel
_logger = logging.getLogger(__name__)
_logger.setLevel(logging.INFO)
# revision identifiers, used by Alembic.
revision = "2b4d017a5e9b"
down_revision = "89d4b8295536"
branch_labels = None
depends_on = None
def upgrade():
bind = op.get_bind()
session = orm.Session(bind=bind)
_logger.info("Adding registered_models and model_versions tables to database.")
op.create_table(
SqlRegisteredModel.__tablename__,
Column("name", String(256), unique=True, nullable=False),
Column("creation_time", BigInteger, default=lambda: int(time.time() * 1000)),
Column("last_updated_time", BigInteger, nullable=True, default=None),
Column("description", String(5000), nullable=True),
PrimaryKeyConstraint("name", name="registered_model_pk"),
)
op.create_table(
SqlModelVersion.__tablename__,
Column("name", String(256), ForeignKey("registered_models.name", onupdate="cascade")),
Column("version", Integer, nullable=False),
Column("creation_time", BigInteger, default=lambda: int(time.time() * 1000)),
Column("last_updated_time", BigInteger, nullable=True, default=None),
Column("description", String(5000), nullable=True),
Column("user_id", String(256), nullable=True, default=None),
Column("current_stage", String(20), default=STAGE_NONE),
Column("source", String(500), nullable=True, default=None),
Column("run_id", String(32), nullable=False),
Column(
"status", String(20), default=ModelVersionStatus.to_string(ModelVersionStatus.READY)
),
Column("status_message", String(500), nullable=True, default=None),
PrimaryKeyConstraint("name", "version", name="model_version_pk"),
)
session.commit()
_logger.info("Migration complete!")
def downgrade():
op.drop_table(SqlRegisteredModel.__tablename__)
op.drop_table(SqlModelVersion.__tablename__)

View File

@@ -0,0 +1,33 @@
"""increase max param val length from 500 to 8000
Revision ID: 2d6e25af4d3e
Revises: 7f2a7d5fae7d
Create Date: 2023-09-25 13:59:04.231744
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "2d6e25af4d3e"
down_revision = "7f2a7d5fae7d"
branch_labels = None
depends_on = None
def upgrade():
with op.batch_alter_table("params") as batch_op:
batch_op.alter_column(
"value",
existing_type=sa.String(500),
# We choose 8000 because it's the minimum max_length for
# a VARCHAR column in all supported database types.
type_=sa.String(8000),
existing_nullable=False,
existing_server_default=None,
)
def downgrade():
pass

View File

@@ -0,0 +1,50 @@
"""Add Model Aliases table
Revision ID: 3500859a5d39
Revises: 97727af70f4d
Create Date: 2023-03-09 15:33:54.951736
"""
import sqlalchemy as sa
from alembic import op
from mlflow.store.model_registry.dbmodels.models import SqlRegisteredModelAlias
# revision identifiers, used by Alembic.
revision = "3500859a5d39"
down_revision = "97727af70f4d"
branch_labels = None
depends_on = None
def get_existing_tables():
connection = op.get_bind()
inspector = sa.inspect(connection)
return inspector.get_table_names()
def upgrade():
if SqlRegisteredModelAlias.__tablename__ not in get_existing_tables():
op.create_table(
SqlRegisteredModelAlias.__tablename__,
sa.Column("alias", sa.String(length=256), primary_key=True, nullable=False),
sa.Column("version", sa.Integer(), nullable=False),
sa.Column(
"name",
sa.String(length=256),
sa.ForeignKey(
"registered_models.name",
onupdate="cascade",
ondelete="cascade",
name="registered_model_alias_name_fkey",
),
primary_key=True,
nullable=False,
),
sa.PrimaryKeyConstraint("name", "alias", name="registered_model_alias_pk"),
)
def downgrade():
pass

View File

@@ -0,0 +1,41 @@
"""add_is_nan_constraint_for_metrics_tables_if_necessary
Revision ID: 39d1c3be5f05
Revises: a8c4a736bde6
Create Date: 2021-03-16 20:40:24.214667
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "39d1c3be5f05"
down_revision = "a8c4a736bde6"
branch_labels = None
depends_on = None
def upgrade():
# This part of the migration is only relevant for users who installed sqlalchemy 1.4.0 with
# MLflow <= 1.14.1. In sqlalchemy 1.4.0, the default value of `create_constraint` for
# `sqlalchemy.Boolean` was changed to `False` from `True`:
# https://github.com/sqlalchemy/sqlalchemy/blob/rel_1_4_0/lib/sqlalchemy/sql/sqltypes.py#L1841.
# To ensure that a check constraint is always present on the `is_nan` column in the
# `latest_metrics` table, we perform an `alter_column` and explicitly set `create_constraint`
# to `True`
with op.batch_alter_table("latest_metrics") as batch_op:
batch_op.alter_column(
"is_nan", type_=sa.types.Boolean(create_constraint=True), nullable=False
)
# Introduce a check constraint on the `is_nan` column from the `metrics` table, which was
# missing prior to this migration
with op.batch_alter_table("metrics") as batch_op:
batch_op.alter_column(
"is_nan", type_=sa.types.Boolean(create_constraint=True), nullable=False
)
def downgrade():
pass

View File

@@ -0,0 +1,38 @@
"""increase max dataset schema size
Revision ID: 4465047574b1
Revises: 5b0e9adcef9c
Create Date: 2024-07-09 12:54:33.775087
"""
import logging
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects.mysql import MEDIUMTEXT
_logger = logging.getLogger(__name__)
# revision identifiers, used by Alembic.
revision = "4465047574b1"
down_revision = "5b0e9adcef9c"
branch_labels = None
depends_on = None
def upgrade():
try:
# For other database backends, the dataset_schema column already satisfies the new length
if op.get_bind().engine.name == "mysql":
op.alter_column("datasets", "dataset_schema", existing_type=sa.TEXT, type_=MEDIUMTEXT)
except Exception as e:
_logger.warning(
"Failed to update dataset_schema column to MEDIUMTEXT type, it may not be supported "
f"by your SQL database. Exception content: {e}"
)
def downgrade():
pass

View File

@@ -0,0 +1,35 @@
"""add metric step
Revision ID: 451aebb31d03
Revises:
Create Date: 2019-04-22 15:29:24.921354
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "451aebb31d03"
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
op.add_column("metrics", sa.Column("step", sa.BigInteger(), nullable=False, server_default="0"))
# Use batch mode so that we can run "ALTER TABLE" statements against SQLite
# databases (see more info at https://alembic.sqlalchemy.org/en/latest/
# batch.html#running-batch-migrations-for-sqlite-and-other-databases)
with op.batch_alter_table("metrics") as batch_op:
batch_op.drop_constraint(constraint_name="metric_pk", type_="primary")
batch_op.create_primary_key(
constraint_name="metric_pk", columns=["key", "timestamp", "step", "run_uuid", "value"]
)
def downgrade():
# This migration cannot safely be downgraded; once metric data with the same
# (key, timestamp, run_uuid, value) are inserted (differing only in their `step`), we cannot
# revert to a schema where (key, timestamp, run_uuid, value) is the metric primary key.
pass

View File

@@ -0,0 +1,40 @@
"""add cascade deletion to trace tables foreign keys
Revision ID: 5b0e9adcef9c
Revises: 867495a8f9d4
Create Date: 2024-05-22 17:44:24.597019
"""
from alembic import op
from mlflow.store.tracking.dbmodels.models import SqlTraceInfo, SqlTraceRequestMetadata, SqlTraceTag
# revision identifiers, used by Alembic.
revision = "5b0e9adcef9c"
down_revision = "867495a8f9d4"
branch_labels = None
depends_on = None
def upgrade():
tables = [SqlTraceTag.__tablename__, SqlTraceRequestMetadata.__tablename__]
for table in tables:
fk_tag_constraint_name = f"fk_{table}_request_id"
# We have to use batch_alter_table as SQLite does not support
# ALTER outside of a batch operation.
with op.batch_alter_table(table, schema=None) as batch_op:
batch_op.drop_constraint(fk_tag_constraint_name, type_="foreignkey")
batch_op.create_foreign_key(
fk_tag_constraint_name,
SqlTraceInfo.__tablename__,
["request_id"],
["request_id"],
# Add cascade deletion to the foreign key constraint.
# This is the only change in this migration.
ondelete="CASCADE",
)
def downgrade():
pass

View File

@@ -0,0 +1,38 @@
"""add registered model tags table
Revision ID: 728d730b5ebd
Revises: 0a8213491aaa
Create Date: 2020-06-26 13:30:00.290154
"""
import sqlalchemy as sa
from alembic import op
from mlflow.store.model_registry.dbmodels.models import SqlRegisteredModelTag
# revision identifiers, used by Alembic.
revision = "728d730b5ebd"
down_revision = "0a8213491aaa"
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
SqlRegisteredModelTag.__tablename__,
sa.Column("key", sa.String(length=250), primary_key=True, nullable=False),
sa.Column("value", sa.String(length=5000)),
sa.Column(
"name",
sa.String(length=256),
sa.ForeignKey("registered_models.name", onupdate="cascade"),
primary_key=True,
nullable=False,
),
sa.PrimaryKeyConstraint("key", "name", name="registered_model_tag_pk"),
)
def downgrade():
pass

View File

@@ -0,0 +1,36 @@
"""Update run tags with larger limit
Revision ID: 7ac759974ad8
Revises: df50e92ffc5e
Create Date: 2019-07-30 16:36:54.256382
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "7ac759974ad8"
down_revision = "df50e92ffc5e"
branch_labels = None
depends_on = None
def upgrade():
# Use batch mode so that we can run "ALTER TABLE" statements against SQLite
# databases (see more info at https://alembic.sqlalchemy.org/en/latest/
# batch.html#running-batch-migrations-for-sqlite-and-other-databases)
# We specify existing_type, existing_nullable, existing_server_default
# because MySQL alter column statements require a full column description.
with op.batch_alter_table("tags") as batch_op:
batch_op.alter_column(
"value",
existing_type=sa.String(250),
type_=sa.String(5000),
existing_nullable=True,
existing_server_default=None,
)
def downgrade():
pass

View File

@@ -0,0 +1,82 @@
"""add datasets inputs input_tags tables
Revision ID: 7f2a7d5fae7d
Revises: 3500859a5d39
Create Date: 2023-03-23 09:48:27.775166
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects.mysql import MEDIUMTEXT
from mlflow.store.tracking.dbmodels.models import SqlDataset, SqlInput, SqlInputTag
# revision identifiers, used by Alembic.
revision = "7f2a7d5fae7d"
down_revision = "3500859a5d39"
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
SqlDataset.__tablename__,
sa.Column("dataset_uuid", sa.String(length=36), nullable=False),
sa.Column(
"experiment_id",
sa.Integer(),
sa.ForeignKey("experiments.experiment_id"),
primary_key=True,
nullable=False,
),
sa.Column("name", sa.String(length=500), primary_key=True, nullable=False),
sa.Column("digest", sa.String(length=36), primary_key=True, nullable=False),
sa.Column("dataset_source_type", sa.String(length=36), nullable=False),
sa.Column("dataset_source", sa.Text(), nullable=False),
sa.Column("dataset_schema", sa.Text(), nullable=True),
sa.Column("dataset_profile", sa.Text().with_variant(MEDIUMTEXT, "mysql"), nullable=True),
sa.PrimaryKeyConstraint("experiment_id", "name", "digest", name="dataset_pk"),
sa.Index(f"index_{SqlDataset.__tablename__}_dataset_uuid", "dataset_uuid", unique=False),
sa.Index(
f"index_{SqlDataset.__tablename__}_experiment_id_dataset_source_type",
"experiment_id",
"dataset_source_type",
unique=False,
),
)
op.create_table(
SqlInput.__tablename__,
sa.Column("input_uuid", sa.String(length=36), nullable=False),
sa.Column("source_type", sa.String(length=36), primary_key=True, nullable=False),
sa.Column("source_id", sa.String(length=36), primary_key=True, nullable=False),
sa.Column("destination_type", sa.String(length=36), primary_key=True, nullable=False),
sa.Column("destination_id", sa.String(length=36), primary_key=True, nullable=False),
sa.PrimaryKeyConstraint(
"source_type", "source_id", "destination_type", "destination_id", name="inputs_pk"
),
sa.Index(f"index_{SqlInput.__tablename__}_input_uuid", "input_uuid", unique=False),
sa.Index(
f"index_{SqlInput.__tablename__}_destination_type_destination_id_source_type",
"destination_type",
"destination_id",
"source_type",
unique=False,
),
)
op.create_table(
SqlInputTag.__tablename__,
sa.Column(
"input_uuid",
sa.String(length=36),
primary_key=True,
nullable=False,
),
sa.Column("name", sa.String(length=255), primary_key=True, nullable=False),
sa.Column("value", sa.String(length=500), nullable=False),
sa.PrimaryKeyConstraint("input_uuid", "name", name="input_tags_pk"),
)
def downgrade():
pass

View File

@@ -0,0 +1,26 @@
"""add run_link to model_version
Revision ID: 84291f40a231
Revises: 27a6a02d2cf1
Create Date: 2020-07-16 13:45:56.178092
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "84291f40a231"
down_revision = "27a6a02d2cf1"
branch_labels = None
depends_on = None
def upgrade():
op.add_column(
"model_versions", sa.Column("run_link", sa.String(500), nullable=True, default=None)
)
def downgrade():
pass

View File

@@ -0,0 +1,90 @@
"""add trace tables
Revision ID: 867495a8f9d4
Revises: acf3f17fdcc7
Create Date: 2024-04-27 12:29:25.178685
"""
import sqlalchemy as sa
from alembic import op
from mlflow.store.tracking.dbmodels.models import SqlTraceInfo, SqlTraceRequestMetadata, SqlTraceTag
# revision identifiers, used by Alembic.
revision = "867495a8f9d4"
down_revision = "acf3f17fdcc7"
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
SqlTraceInfo.__tablename__,
sa.Column("request_id", sa.String(length=50), primary_key=True, nullable=False),
sa.Column(
"experiment_id",
sa.Integer(),
sa.ForeignKey(
column="experiments.experiment_id",
name="fk_trace_info_experiment_id",
),
nullable=False,
),
sa.Column("timestamp_ms", sa.BigInteger(), nullable=False),
sa.Column("execution_time_ms", sa.BigInteger(), nullable=True),
sa.Column("status", sa.String(length=50), nullable=False),
sa.PrimaryKeyConstraint("request_id", name="trace_info_pk"),
sa.Index(
f"index_{SqlTraceInfo.__tablename__}_experiment_id_timestamp_ms",
"experiment_id",
"timestamp_ms",
unique=False,
),
)
op.create_table(
SqlTraceTag.__tablename__,
sa.Column("key", sa.String(length=250), primary_key=True, nullable=False),
sa.Column("value", sa.String(length=8000), nullable=True),
sa.Column(
"request_id",
sa.String(length=50),
sa.ForeignKey(
column=SqlTraceInfo.request_id,
name=f"fk_{SqlTraceTag.__tablename__}_request_id",
),
nullable=False,
primary_key=True,
),
sa.PrimaryKeyConstraint("key", "request_id", name="trace_tag_pk"),
sa.Index(
f"index_{SqlTraceTag.__tablename__}_request_id",
"request_id",
unique=False,
),
)
op.create_table(
SqlTraceRequestMetadata.__tablename__,
sa.Column("key", sa.String(length=250), primary_key=True, nullable=False),
sa.Column("value", sa.String(length=8000), nullable=True),
sa.Column(
"request_id",
sa.String(length=50),
sa.ForeignKey(
column=SqlTraceInfo.request_id,
name=f"fk_{SqlTraceRequestMetadata.__tablename__}_request_id",
),
nullable=False,
primary_key=True,
),
sa.PrimaryKeyConstraint("key", "request_id", name="trace_request_metadata_pk"),
sa.Index(
f"index_{SqlTraceRequestMetadata.__tablename__}_request_id",
"request_id",
unique=False,
),
)
def downgrade():
pass

View File

@@ -0,0 +1,169 @@
"""create latest metrics table
Revision ID: 89d4b8295536
Revises: 7ac759974ad8
Create Date: 2019-08-20 11:53:28.178479
"""
import logging
import time
from alembic import op
from sqlalchemy import (
BigInteger,
Boolean,
Column,
Float,
ForeignKey,
PrimaryKeyConstraint,
String,
and_,
distinct,
func,
orm,
)
from mlflow.store.tracking.dbmodels.models import SqlLatestMetric, SqlMetric
_logger = logging.getLogger(__name__)
_logger.setLevel(logging.INFO)
# revision identifiers, used by Alembic.
revision = "89d4b8295536"
down_revision = "7ac759974ad8"
branch_labels = None
depends_on = None
def _describe_migration_if_necessary(session):
"""
If the targeted database contains any metric entries, this function emits important,
database-specific information about the ``create_latest_metrics_table`` migration.
If the targeted database does *not* contain any metric entries, this output is omitted
in order to avoid superfluous log output when initializing a new Tracking database.
"""
num_metric_entries = session.query(SqlMetric).count()
if num_metric_entries <= 0:
return
_logger.warning(
"**IMPORTANT**: This migration creates a `latest_metrics` table and populates it with the"
" latest metric entry for each unique (run_id, metric_key) tuple. Latest metric entries are"
" computed based on step, timestamp, and value. This migration may take a long time for"
" databases containing a large number of metric entries. Please refer to {readme_link} for"
" information about this migration, including how to estimate migration size and how to"
" restore your database to its original state if the migration is unsuccessful. If you"
" encounter failures while executing this migration, please file a GitHub issue at"
" {issues_link}.".format(
readme_link=(
"https://github.com/mlflow/mlflow/blob/master/mlflow/store/db_migrations/README.md"
"#89d4b8295536_create_latest_metrics_table"
),
issues_link="https://github.com/mlflow/mlflow/issues",
)
)
num_metric_keys = (
session.query(SqlMetric.run_uuid, SqlMetric.key)
.group_by(SqlMetric.run_uuid, SqlMetric.key)
.count()
)
num_runs_containing_metrics = session.query(distinct(SqlMetric.run_uuid)).count()
_logger.info(
"This tracking database has {num_metric_entries} total metric entries for {num_metric_keys}"
" unique metrics across {num_runs} runs.".format(
num_metric_entries=num_metric_entries,
num_metric_keys=num_metric_keys,
num_runs=num_runs_containing_metrics,
)
)
def _get_latest_metrics_for_runs(session):
metrics_with_max_step = (
session.query(SqlMetric.run_uuid, SqlMetric.key, func.max(SqlMetric.step).label("step"))
.group_by(SqlMetric.key, SqlMetric.run_uuid)
.subquery("metrics_with_max_step")
)
metrics_with_max_timestamp = (
session.query(
SqlMetric.run_uuid,
SqlMetric.key,
SqlMetric.step,
func.max(SqlMetric.timestamp).label("timestamp"),
)
.join(
metrics_with_max_step,
and_(
SqlMetric.step == metrics_with_max_step.c.step,
SqlMetric.run_uuid == metrics_with_max_step.c.run_uuid,
SqlMetric.key == metrics_with_max_step.c.key,
),
)
.group_by(SqlMetric.key, SqlMetric.run_uuid, SqlMetric.step)
.subquery("metrics_with_max_timestamp")
)
return (
session.query(
SqlMetric.run_uuid,
SqlMetric.key,
SqlMetric.step,
SqlMetric.timestamp,
func.max(SqlMetric.value).label("value"),
SqlMetric.is_nan,
)
.join(
metrics_with_max_timestamp,
and_(
SqlMetric.timestamp == metrics_with_max_timestamp.c.timestamp,
SqlMetric.run_uuid == metrics_with_max_timestamp.c.run_uuid,
SqlMetric.key == metrics_with_max_timestamp.c.key,
SqlMetric.step == metrics_with_max_timestamp.c.step,
),
)
.group_by(
SqlMetric.run_uuid, SqlMetric.key, SqlMetric.step, SqlMetric.timestamp, SqlMetric.is_nan
)
.all()
)
def upgrade():
bind = op.get_bind()
session = orm.Session(bind=bind)
_describe_migration_if_necessary(session)
all_latest_metrics = _get_latest_metrics_for_runs(session=session)
op.create_table(
SqlLatestMetric.__tablename__,
Column("key", String(length=250)),
Column("value", Float(precision=53), nullable=False),
Column("timestamp", BigInteger, default=lambda: int(time.time())),
Column("step", BigInteger, default=0, nullable=False),
Column("is_nan", Boolean, default=False, nullable=False),
Column("run_uuid", String(length=32), ForeignKey("runs.run_uuid"), nullable=False),
PrimaryKeyConstraint("key", "run_uuid", name="latest_metric_pk"),
)
session.add_all(
[
SqlLatestMetric(
run_uuid=run_uuid,
key=key,
step=step,
timestamp=timestamp,
value=value,
is_nan=is_nan,
)
for run_uuid, key, step, timestamp, value, is_nan in all_latest_metrics
]
)
session.commit()
_logger.info("Migration complete!")
def downgrade():
op.drop_table(SqlLatestMetric.__tablename__)

View File

@@ -0,0 +1,64 @@
"""migrate user column to tags
Revision ID: 90e64c465722
Revises: 451aebb31d03
Create Date: 2019-05-29 10:43:52.919427
"""
from alembic import op
from sqlalchemy import Column, ForeignKey, Integer, PrimaryKeyConstraint, String, orm
from sqlalchemy.orm import backref, declarative_base, relationship
from mlflow.utils.mlflow_tags import MLFLOW_USER
# revision identifiers, used by Alembic.
revision = "90e64c465722"
down_revision = "451aebb31d03"
branch_labels = None
depends_on = None
Base = declarative_base()
class SqlRun(Base):
__tablename__ = "runs"
run_uuid = Column(String(32), nullable=False)
user_id = Column(String(256), nullable=True, default=None)
experiment_id = Column(Integer)
__table_args__ = (PrimaryKeyConstraint("experiment_id", name="experiment_pk"),)
class SqlTag(Base):
__tablename__ = "tags"
key = Column(String(250))
value = Column(String(250), nullable=True)
run_uuid = Column(String(32), ForeignKey("runs.run_uuid"))
run = relationship("SqlRun", backref=backref("tags", cascade="all"))
__table_args__ = (PrimaryKeyConstraint("key", "run_uuid", name="tag_pk"),)
def upgrade():
bind = op.get_bind()
session = orm.Session(bind=bind)
runs = session.query(SqlRun).all()
for run in runs:
if not run.user_id:
continue
tag_exists = False
for tag in run.tags:
if tag.key == MLFLOW_USER:
tag_exists = True
if tag_exists:
continue
session.merge(SqlTag(run_uuid=run.run_uuid, key=MLFLOW_USER, value=run.user_id))
session.commit()
def downgrade():
pass

View File

@@ -0,0 +1,25 @@
"""Add creation_time and last_update_time to experiments table
Revision ID: 97727af70f4d
Revises: cc1f77228345
Create Date: 2022-08-26 21:16:59.164858
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "97727af70f4d"
down_revision = "cc1f77228345"
branch_labels = None
depends_on = None
def upgrade():
op.add_column("experiments", sa.Column("creation_time", sa.BigInteger(), nullable=True))
op.add_column("experiments", sa.Column("last_update_time", sa.BigInteger(), nullable=True))
def downgrade():
pass

View File

@@ -0,0 +1,27 @@
"""allow nulls for run_id
Revision ID: a8c4a736bde6
Revises: 84291f40a231
Create Date: 2020-12-02 12:14:35.220815
"""
import sqlalchemy as sa
from alembic import op
from mlflow.store.model_registry.dbmodels.models import SqlModelVersion
# revision identifiers, used by Alembic.
revision = "a8c4a736bde6"
down_revision = "84291f40a231"
branch_labels = None
depends_on = None
def upgrade():
with op.batch_alter_table(SqlModelVersion.__tablename__) as batch_op:
batch_op.alter_column("run_id", nullable=True, existing_type=sa.VARCHAR(32))
def downgrade():
pass

View File

@@ -0,0 +1,29 @@
"""add storage location field to model versions
Revision ID: acf3f17fdcc7
Revises: 2d6e25af4d3e
Create Date: 2023-10-23 15:26:53.062080
"""
import sqlalchemy as sa
from alembic import op
from mlflow.store.model_registry.dbmodels.models import SqlModelVersion
# revision identifiers, used by Alembic.
revision = "acf3f17fdcc7"
down_revision = "2d6e25af4d3e"
branch_labels = None
depends_on = None
def upgrade():
op.add_column(
SqlModelVersion.__tablename__,
sa.Column("storage_location", sa.String(500), nullable=True, default=None),
)
def downgrade():
pass

View File

@@ -0,0 +1,26 @@
"""create index on run_uuid
Revision ID: bd07f7e963c5
Revises: c48cb773bb87
Create Date: 2022-03-03 10:14:34.037978
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = "bd07f7e963c5"
down_revision = "c48cb773bb87"
branch_labels = None
depends_on = None
def upgrade():
# As a fix for https://github.com/mlflow/mlflow/issues/3785, create an index on run_uuid columns
# that have a foreign key constraint to speed up SQL operations.
for table in ["params", "metrics", "latest_metrics", "tags"]:
op.create_index(f"index_{table}_run_uuid", table, ["run_uuid"])
def downgrade():
pass

View File

@@ -0,0 +1,41 @@
"""reset_default_value_for_is_nan_in_metrics_table_for_mysql
Revision ID: c48cb773bb87
Revises: 39d1c3be5f05
Create Date: 2021-04-02 15:43:28.466043
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "c48cb773bb87"
down_revision = "39d1c3be5f05"
branch_labels = None
depends_on = None
def upgrade():
# This part of the migration is only relevant for MySQL.
# In 39d1c3be5f05_add_is_nan_constraint_for_metrics_tables_if_necessary.py
# (added in MLflow 1.15.0), `alter_column` is called on the `is_nan` column in the `metrics`
# table without specifying `existing_server_default`. This alters the column default value to
# NULL in MySQL (see the doc below).
#
# https://alembic.sqlalchemy.org/en/latest/ops.html#alembic.operations.Operations.alter_column
#
# To revert this change, set the default column value to "0" by specifying `server_default`
bind = op.get_bind()
if bind.engine.name == "mysql":
with op.batch_alter_table("metrics") as batch_op:
batch_op.alter_column(
"is_nan",
type_=sa.types.Boolean(create_constraint=True),
nullable=False,
server_default="0",
)
def downgrade():
pass

View File

@@ -0,0 +1,34 @@
"""change param value length to 500
Revision ID: cc1f77228345
Revises: 0c779009ac13
Create Date: 2022-08-04 22:40:56.960003
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "cc1f77228345"
down_revision = "0c779009ac13"
branch_labels = None
depends_on = None
def upgrade():
"""
Enlarge the maximum param value length to 500.
"""
with op.batch_alter_table("params") as batch_op:
batch_op.alter_column(
"value",
existing_type=sa.String(250),
type_=sa.String(500),
existing_nullable=False,
nullable=False,
)
def downgrade():
pass

View File

@@ -0,0 +1,78 @@
"""Update run status constraint with killed
Revision ID: cfd24bdc0731
Revises: 89d4b8295536
Create Date: 2019-10-11 15:55:10.853449
"""
import alembic
from alembic import op
from packaging.version import Version
from sqlalchemy import CheckConstraint, Enum
from mlflow.entities import RunStatus, ViewType
from mlflow.entities.lifecycle_stage import LifecycleStage
from mlflow.store.tracking.dbmodels.models import SourceTypes, SqlRun
# revision identifiers, used by Alembic.
revision = "cfd24bdc0731"
down_revision = "2b4d017a5e9b"
branch_labels = None
depends_on = None
old_run_statuses = [
RunStatus.to_string(RunStatus.SCHEDULED),
RunStatus.to_string(RunStatus.FAILED),
RunStatus.to_string(RunStatus.FINISHED),
RunStatus.to_string(RunStatus.RUNNING),
]
new_run_statuses = [*old_run_statuses, RunStatus.to_string(RunStatus.KILLED)]
# Certain SQL backends (e.g., SQLite) do not preserve CHECK constraints during migrations.
# For these backends, CHECK constraints must be specified as table arguments. Here, we define
# the collection of CHECK constraints that should be preserved when performing the migration.
# The "status" constraint is excluded from this set because it is explicitly modified
# within the migration's `upgrade()` routine.
check_constraint_table_args = [
CheckConstraint(SqlRun.source_type.in_(SourceTypes), name="source_type"),
CheckConstraint(
SqlRun.lifecycle_stage.in_(LifecycleStage.view_type_to_stages(ViewType.ALL)),
name="runs_lifecycle_stage",
),
]
def upgrade():
# In alembic >= 1.7.0, `table_args` is unnecessary since CHECK constraints are preserved
# during migrations.
table_args = (
[] if Version(alembic.__version__) >= Version("1.7.0") else check_constraint_table_args
)
with op.batch_alter_table("runs", table_args=table_args) as batch_op:
# Transform the "status" column to an `Enum` and define a new check constraint. Specify
# `native_enum=False` to create a check constraint rather than a
# database-backend-dependent enum (see https://docs.sqlalchemy.org/en/13/core/
# type_basics.html#sqlalchemy.types.Enum.params.native_enum)
batch_op.alter_column(
"status",
type_=Enum(
*new_run_statuses,
create_constraint=True,
native_enum=False,
),
existing_type=Enum(
*old_run_statuses,
create_constraint=True,
native_enum=False,
name="status",
),
)
def downgrade():
# Omit downgrade logic for now - we don't currently provide users a command/API for
# reverting a database migration, instead recommending that they take a database backup
# before running the migration.
pass

View File

@@ -0,0 +1,38 @@
"""Add Experiment Tags Table
Revision ID: df50e92ffc5e
Revises: 181f10493468
Create Date: 2019-07-15 17:46:42.704214
"""
import sqlalchemy as sa
from alembic import op
from mlflow.store.tracking.dbmodels.models import SqlExperimentTag
# revision identifiers, used by Alembic.
revision = "df50e92ffc5e"
down_revision = "181f10493468"
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
SqlExperimentTag.__tablename__,
sa.Column("key", sa.String(length=250), primary_key=True, nullable=False),
sa.Column("value", sa.String(length=5000)),
sa.Column(
"experiment_id",
sa.Integer(),
sa.ForeignKey("experiments.experiment_id"),
primary_key=True,
nullable=False,
),
sa.PrimaryKeyConstraint("key", "experiment_id", name="experiment_tag_pk"),
)
def downgrade():
pass

View File

@@ -0,0 +1,36 @@
"""increase run tag value limit to 8000
Revision ID: f5a4f2784254
Revises: 4465047574b1
Create Date: 2024-09-18 08:53:51.552934
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "f5a4f2784254"
down_revision = "4465047574b1"
branch_labels = None
depends_on = None
def upgrade():
# Use batch mode so that we can run "ALTER TABLE" statements against SQLite
# databases (see more info at https://alembic.sqlalchemy.org/en/latest/
# batch.html#running-batch-migrations-for-sqlite-and-other-databases)
# We specify existing_type, existing_nullable, existing_server_default
# because MySQL alter column statements require a full column description.
with op.batch_alter_table("tags") as batch_op:
batch_op.alter_column(
"value",
existing_type=sa.String(5000),
type_=sa.String(8000),
existing_nullable=True,
existing_server_default=None,
)
def downgrade():
pass