diff --git a/config/config.exs b/config/config.exs index ee88e028..0041c0ac 100644 --- a/config/config.exs +++ b/config/config.exs @@ -37,6 +37,14 @@ if Mix.env() == :test do hostname: "localhost", pool: Ecto.Adapters.SQL.Sandbox + config :ash_postgres, AshPostgres.DevTestRepo, + username: "postgres", + password: "postgres", + database: "ash_postgres_dev_test", + hostname: "localhost", + migration_primary_key: [name: :id, type: :binary_id], + pool: Ecto.Adapters.SQL.Sandbox + # sobelow_skip ["Config.Secrets"] config :ash_postgres, AshPostgres.TestRepo, password: "postgres" @@ -54,7 +62,7 @@ if Mix.env() == :test do migration_primary_key: [name: :id, type: :binary_id] config :ash_postgres, - ecto_repos: [AshPostgres.TestRepo, AshPostgres.TestNoSandboxRepo], + ecto_repos: [AshPostgres.TestRepo, AshPostgres.DevTestRepo, AshPostgres.TestNoSandboxRepo], ash_domains: [ AshPostgres.Test.Domain, AshPostgres.MultitenancyTest.Domain, diff --git a/lib/migration_generator/migration_generator.ex b/lib/migration_generator/migration_generator.ex index 73fe1d3d..2ab6706e 100644 --- a/lib/migration_generator/migration_generator.ex +++ b/lib/migration_generator/migration_generator.ex @@ -19,6 +19,7 @@ defmodule AshPostgres.MigrationGenerator do format: true, dry_run: false, check: false, + dev: false, snapshots_only: false, dont_drop_columns: false @@ -452,6 +453,23 @@ defmodule AshPostgres.MigrationGenerator do :ok operations -> + dev_migrations = get_dev_migrations(opts, tenant?, repo) + + if !opts.dev and dev_migrations != [] do + if opts.check do + Mix.shell().error(""" + Generated migrations are from dev mode. + + Generate migrations without `--dev` flag. + """) + + exit({:shutdown, 1}) + else + remove_dev_migrations(dev_migrations, tenant?, repo, opts) + remove_dev_snapshots(snapshots, opts) + end + end + if opts.check do Mix.shell().error(""" Migrations would have been generated, but the --check flag was provided. @@ -491,6 +509,46 @@ defmodule AshPostgres.MigrationGenerator do end) end + defp get_dev_migrations(opts, tenant?, repo) do + opts + |> migration_path(repo, tenant?) + |> File.ls() + |> case do + {:error, _error} -> [] + {:ok, migrations} -> Enum.filter(migrations, &String.contains?(&1, "_dev.exs")) + end + end + + defp remove_dev_migrations(dev_migrations, tenant?, repo, opts) do + version = dev_migrations |> Enum.min() |> String.split("_") |> hd() + Mix.Task.reenable("ash_postgres.rollback") + Mix.Task.run("ash_postgres.rollback", ["--repo", inspect(repo), "--to", version]) + + dev_migrations + |> Enum.each(fn migration_name -> + opts + |> migration_path(repo, tenant?) + |> Path.join(migration_name) + |> File.rm!() + end) + end + + def remove_dev_snapshots(snapshots, opts) do + Enum.each(snapshots, fn snapshot -> + folder = get_snapshot_folder(snapshot, opts) + snapshot_path = get_snapshot_path(snapshot, folder) + + snapshot_path + |> File.ls!() + |> Enum.filter(&String.contains?(&1, "_dev.json")) + |> Enum.each(fn snapshot_name -> + snapshot_path + |> Path.join(snapshot_name) + |> File.rm!() + end) + end) + end + defp split_into_migrations(operations) do operations |> Enum.split_with(fn @@ -960,7 +1018,7 @@ defmodule AshPostgres.MigrationGenerator do migration_file = migration_path - |> Path.join(migration_name <> ".exs") + |> Path.join(migration_name <> "#{if opts.dev, do: "_dev"}.exs") module_name = if tenant? do @@ -1054,20 +1112,25 @@ defmodule AshPostgres.MigrationGenerator do |> Path.join(repo_name) end + dev = if opts.dev, do: "_dev" + snapshot_file = if snapshot.schema do - Path.join(snapshot_folder, "#{snapshot.schema}.#{snapshot.table}/#{timestamp()}.json") + Path.join( + snapshot_folder, + "#{snapshot.schema}.#{snapshot.table}/#{timestamp()}#{dev}.json" + ) else - Path.join(snapshot_folder, "#{snapshot.table}/#{timestamp()}.json") + Path.join(snapshot_folder, "#{snapshot.table}/#{timestamp()}#{dev}.json") end File.mkdir_p(Path.dirname(snapshot_file)) create_file(snapshot_file, snapshot_binary, force: true) - old_snapshot_folder = Path.join(snapshot_folder, "#{snapshot.table}.json") + old_snapshot_folder = Path.join(snapshot_folder, "#{snapshot.table}#{dev}.json") if File.exists?(old_snapshot_folder) do - new_snapshot_folder = Path.join(snapshot_folder, "#{snapshot.table}/initial.json") + new_snapshot_folder = Path.join(snapshot_folder, "#{snapshot.table}/initial#{dev}.json") File.rename(old_snapshot_folder, new_snapshot_folder) end end) @@ -2623,43 +2686,22 @@ defmodule AshPostgres.MigrationGenerator do end def get_existing_snapshot(snapshot, opts) do - repo_name = snapshot.repo |> Module.split() |> List.last() |> Macro.underscore() - - folder = - if snapshot.multitenancy.strategy == :context do - opts - |> snapshot_path(snapshot.repo) - |> Path.join(repo_name) - |> Path.join("tenants") - else - opts - |> snapshot_path(snapshot.repo) - |> Path.join(repo_name) - end + folder = get_snapshot_folder(snapshot, opts) + snapshot_path = get_snapshot_path(snapshot, folder) - snapshot_folder = - if snapshot.schema do - schema_dir = Path.join(folder, "#{snapshot.schema}.#{snapshot.table}") - - if File.dir?(schema_dir) do - schema_dir - else - Path.join(folder, snapshot.table) - end - else - Path.join(folder, snapshot.table) - end - - if File.exists?(snapshot_folder) do - snapshot_folder + if File.exists?(snapshot_path) do + snapshot_path |> File.ls!() - |> Enum.filter(&String.match?(&1, ~r/^\d{14}\.json$/)) + |> Enum.filter( + &(String.match?(&1, ~r/^\d{14}\.json$/) or + (opts.dev and String.match?(&1, ~r/^\d{14}\_dev.json$/))) + ) |> case do [] -> get_old_snapshot(folder, snapshot) snapshot_files -> - snapshot_folder + snapshot_path |> Path.join(Enum.max(snapshot_files)) |> File.read!() |> load_snapshot() @@ -2669,6 +2711,33 @@ defmodule AshPostgres.MigrationGenerator do end end + defp get_snapshot_folder(snapshot, opts) do + if snapshot.multitenancy.strategy == :context do + opts + |> snapshot_path(snapshot.repo) + |> Path.join(repo_name(snapshot.repo)) + |> Path.join("tenants") + else + opts + |> snapshot_path(snapshot.repo) + |> Path.join(repo_name(snapshot.repo)) + end + end + + defp get_snapshot_path(snapshot, folder) do + if snapshot.schema do + schema_dir = Path.join(folder, "#{snapshot.schema}.#{snapshot.table}") + + if File.dir?(schema_dir) do + schema_dir + else + Path.join(folder, snapshot.table) + end + else + Path.join(folder, snapshot.table) + end + end + defp get_old_snapshot(folder, snapshot) do schema_file = if snapshot.schema do diff --git a/lib/mix/tasks/ash_postgres.generate_migrations.ex b/lib/mix/tasks/ash_postgres.generate_migrations.ex index 065942c8..791f0703 100644 --- a/lib/mix/tasks/ash_postgres.generate_migrations.ex +++ b/lib/mix/tasks/ash_postgres.generate_migrations.ex @@ -21,6 +21,7 @@ defmodule Mix.Tasks.AshPostgres.GenerateMigrations do * `no-format` - files that are created will not be formatted with the code formatter * `dry-run` - no files are created, instead the new migration is printed * `check` - no files are created, returns an exit(1) code if the current snapshots and resources don't fit + * `dev` - dev files are created * `snapshots-only` - no migrations are generated, only snapshots are stored * `concurrent-indexes` - new identities will be run concurrently and in a separate migration (like concurrent custom indexes) @@ -97,6 +98,7 @@ defmodule Mix.Tasks.AshPostgres.GenerateMigrations do no_format: :boolean, dry_run: :boolean, check: :boolean, + dev: :boolean, dont_drop_columns: :boolean, concurrent_indexes: :boolean ] @@ -110,9 +112,9 @@ defmodule Mix.Tasks.AshPostgres.GenerateMigrations do |> Keyword.delete(:no_format) |> Keyword.put_new(:name, name) - if !opts[:name] && !opts[:dry_run] && !opts[:check] && !opts[:snapshots_only] do + if !opts[:name] && !opts[:dry_run] && !opts[:check] && !opts[:snapshots_only] && !opts[:dev] do IO.warn(""" - Name must be provided when generating migrations, unless `--dry-run` or `--check` is also provided. + Name must be provided when generating migrations, unless `--dry-run` or `--check` or `--dev` is also provided. Using an autogenerated name will be deprecated in a future release. Please provide a name. for example: diff --git a/test/dev_migrations_test.exs b/test/dev_migrations_test.exs new file mode 100644 index 00000000..74f8112c --- /dev/null +++ b/test/dev_migrations_test.exs @@ -0,0 +1,212 @@ +defmodule AshPostgres.DevMigrationsTest do + use AshPostgres.RepoCase, async: false + @moduletag :migration + + import ExUnit.CaptureLog + + alias Ecto.Adapters.SQL.Sandbox + + setup do + current_shell = Mix.shell() + + :ok = Mix.shell(Mix.Shell.Process) + + on_exit(fn -> + Mix.shell(current_shell) + end) + + Sandbox.checkout(AshPostgres.DevTestRepo) + end + + defmacrop defresource(mod, do: body) do + quote do + Code.compiler_options(ignore_module_conflict: true) + + defmodule unquote(mod) do + use Ash.Resource, + domain: nil, + data_layer: AshPostgres.DataLayer + + unquote(body) + end + + Code.compiler_options(ignore_module_conflict: false) + end + end + + defmacrop defposts(mod \\ Post, do: body) do + quote do + defresource unquote(mod) do + postgres do + table "posts" + repo(AshPostgres.DevTestRepo) + + custom_indexes do + # need one without any opts + index(["id"]) + index(["id"], unique: true, name: "test_unique_index") + end + end + + actions do + defaults([:create, :read, :update, :destroy]) + end + + unquote(body) + end + end + end + + defmacrop defdomain(resources) do + quote do + Code.compiler_options(ignore_module_conflict: true) + + defmodule Domain do + use Ash.Domain + + resources do + for resource <- unquote(resources) do + resource(resource) + end + end + end + + Code.compiler_options(ignore_module_conflict: false) + end + end + + describe "--dev option" do + setup do + on_exit(fn -> + resource_dev_path = "priv/resource_snapshots/dev_test_repo" + resource_files = File.ls!(resource_dev_path) + Enum.each(resource_files, &File.rm_rf!(Path.join(resource_dev_path, &1))) + migrations_dev_path = "priv/dev_test_repo/migrations" + migration_files = File.ls!(migrations_dev_path) + Enum.each(migration_files, &File.rm!(Path.join(migrations_dev_path, &1))) + tenant_migrations_dev_path = "priv/dev_test_repo/tenant_migrations" + tenant_migration_files = File.ls!(tenant_migrations_dev_path) + Enum.each(tenant_migration_files, &File.rm!(Path.join(tenant_migrations_dev_path, &1))) + AshPostgres.DevTestRepo.query!("DROP TABLE posts") + end) + end + + test "rolls back dev migrations before deleting" do + defposts do + attributes do + uuid_primary_key(:id) + attribute(:title, :string, public?: true) + end + end + + defdomain([Post]) + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "priv/resource_snapshots", + migration_path: "priv/dev_test_repo/migrations", + dev: true + ) + + assert [_migration] = + Enum.sort( + Path.wildcard("priv/dev_test_repo/migrations/**/*_migrate_resources*.exs") + ) + |> Enum.reject(&String.contains?(&1, "extensions")) + + capture_log(fn -> migrate() end) =~ "create table posts" + capture_log(fn -> migrate() end) =~ "create table posts" + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "priv/resource_snapshots", + migration_path: "priv/dev_test_repo/migrations" + ) + + capture_log(fn -> migrate() end) =~ "create table posts" + end + end + + describe "--dev option tenant" do + setup do + on_exit(fn -> + resource_dev_path = "priv/resource_snapshots/dev_test_repo" + resource_files = File.ls!(resource_dev_path) + Enum.each(resource_files, &File.rm_rf!(Path.join(resource_dev_path, &1))) + migrations_dev_path = "priv/dev_test_repo/migrations" + migration_files = File.ls!(migrations_dev_path) + Enum.each(migration_files, &File.rm!(Path.join(migrations_dev_path, &1))) + tenant_migrations_dev_path = "priv/dev_test_repo/tenant_migrations" + tenant_migration_files = File.ls!(tenant_migrations_dev_path) + Enum.each(tenant_migration_files, &File.rm!(Path.join(tenant_migrations_dev_path, &1))) + end) + end + + test "rolls back dev migrations before deleting" do + defposts do + postgres do + schema("example") + end + + attributes do + uuid_primary_key(:id) + attribute(:title, :string, public?: true, primary_key?: true, allow_nil?: false) + end + + multitenancy do + strategy(:context) + end + end + + defdomain([Post]) + capture_log(fn -> tenant_migrate() end) |> dbg() + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "priv/resource_snapshots", + migration_path: "priv/dev_test_repo/migrations", + tenant_migration_path: "priv/dev_test_repo/tenant_migrations", + dev: true + ) + + assert [] = + Enum.sort( + Path.wildcard("priv/dev_test_repo/migrations/**/*_migrate_resources*.exs") + ) + |> Enum.reject(&String.contains?(&1, "extensions")) + + assert [_tenant_migration] = + Enum.sort( + Path.wildcard("priv/dev_test_repo/tenant_migrations/**/*_migrate_resources*.exs") + ) + |> Enum.reject(&String.contains?(&1, "extensions")) + + assert capture_log(fn -> tenant_migrate() end) =~ "create table posts" + assert capture_log(fn -> tenant_migrate() end) =~ "create table posts" + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "priv/resource_snapshots", + migration_path: "priv/dev_test_repo/migrations", + tenant_migration_path: "priv/dev_test_repo/tenant_migrations" + ) + + assert capture_log(fn -> tenant_migrate() end) =~ "create table posts" + end + end + + defp migrate do + Mix.Tasks.AshPostgres.Migrate.run([ + "--migrations-path", + "priv/dev_test_repo/migrations", + "--repo", + "AshPostgres.DevTestRepo" + ]) + end + + defp tenant_migrate do + Mix.Tasks.AshPostgres.Migrate.run([ + "--migrations-path", + "priv/dev_test_repo/tenant_migrations", + "--repo", + "AshPostgres.DevTestRepo", + "--tenants" + ]) + end +end diff --git a/test/migration_generator_test.exs b/test/migration_generator_test.exs index 8c613055..74d5ffa2 100644 --- a/test/migration_generator_test.exs +++ b/test/migration_generator_test.exs @@ -1326,6 +1326,102 @@ defmodule AshPostgres.MigrationGeneratorTest do end end + describe "--dev option" do + setup do + on_exit(fn -> + File.rm_rf!("test_snapshots_path") + File.rm_rf!("test_migration_path") + File.rm_rf!("test_tenant_migration_path") + end) + end + + test "generates dev migration" do + defposts do + attributes do + uuid_primary_key(:id) + attribute(:title, :string, public?: true) + end + end + + defdomain([Post]) + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "test_snapshots_path", + migration_path: "test_migration_path", + dev: true + ) + + assert [dev_file] = + Path.wildcard("test_migration_path/**/*_migrate_resources*.exs") + |> Enum.reject(&String.contains?(&1, "extensions")) + + assert String.contains?(dev_file, "_dev.exs") + contents = File.read!(dev_file) + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "test_snapshots_path", + migration_path: "test_migration_path" + ) + + assert [file] = + Path.wildcard("test_migration_path/**/*_migrate_resources*.exs") + |> Enum.reject(&String.contains?(&1, "extensions")) + + refute String.contains?(file, "_dev.exs") + + assert contents == File.read!(file) + end + + test "generates dev migration for tenant" do + defposts do + postgres do + schema("example") + end + + attributes do + uuid_primary_key(:id) + attribute(:title, :string, public?: true, primary_key?: true, allow_nil?: false) + end + + multitenancy do + strategy(:context) + end + end + + defdomain([Post]) + + send(self(), {:mix_shell_input, :yes?, true}) + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "test_snapshots_path", + migration_path: "test_migration_path", + tenant_migration_path: "test_tenant_migration_path", + dev: true + ) + + assert [dev_file] = + Enum.sort(Path.wildcard("test_tenant_migration_path/**/*_migrate_resources*.exs")) + |> Enum.reject(&String.contains?(&1, "extensions")) + + assert String.contains?(dev_file, "_dev.exs") + contents = File.read!(dev_file) + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "test_snapshots_path", + migration_path: "test_migration_path", + tenant_migration_path: "test_tenant_migration_path" + ) + + assert [file] = + Path.wildcard("test_tenant_migration_path/**/*_migrate_resources*.exs") + |> Enum.reject(&String.contains?(&1, "extensions")) + + refute String.contains?(file, "_dev.exs") + + assert contents == File.read!(file) + end + end + describe "references" do setup do on_exit(fn -> diff --git a/test/support/dev_test_repo.ex b/test/support/dev_test_repo.ex new file mode 100644 index 00000000..67819081 --- /dev/null +++ b/test/support/dev_test_repo.ex @@ -0,0 +1,39 @@ +defmodule AshPostgres.DevTestRepo do + @moduledoc false + use AshPostgres.Repo, + otp_app: :ash_postgres + + def on_transaction_begin(data) do + send(self(), data) + end + + def prefer_transaction?, do: false + + def prefer_transaction_for_atomic_updates?, do: false + + def installed_extensions do + ["ash-functions", "uuid-ossp", "pg_trgm", "citext", AshPostgres.TestCustomExtension, "ltree"] -- + Application.get_env(:ash_postgres, :no_extensions, []) + end + + def min_pg_version do + case System.get_env("PG_VERSION") do + nil -> + %Version{major: 16, minor: 0, patch: 0} + + version -> + case Integer.parse(version) do + {major, ""} -> %Version{major: major, minor: 0, patch: 0} + _ -> Version.parse!(version) + end + end + end + + def all_tenants do + Code.ensure_compiled(AshPostgres.MultitenancyTest.Org) + + AshPostgres.MultitenancyTest.Org + |> Ash.read!() + |> Enum.map(&"org_#{&1.id}") + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 056ebefd..6745f6eb 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -20,6 +20,7 @@ exclude_tags = ExUnit.configure(stacktrace_depth: 100, exclude: exclude_tags) AshPostgres.TestRepo.start_link() +AshPostgres.DevTestRepo.start_link() AshPostgres.TestNoSandboxRepo.start_link() format_sql_query = @@ -49,4 +50,5 @@ format_sql_query = end Ecto.DevLogger.install(AshPostgres.TestRepo, before_inline_callback: format_sql_query) +Ecto.DevLogger.install(AshPostgres.DevTestRepo, before_inline_callback: format_sql_query) Ecto.DevLogger.install(AshPostgres.TestNoSandboxRepo, before_inline_callback: format_sql_query)