1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309
|
#!/usr/bin/env ruby
# frozen_string_literal: true
ENV['RAILS_ENV'] = 'test'
require 'optparse'
require 'open3'
require 'fileutils'
require 'uri'
class SchemaRegenerator
##
# Filename of the schema
#
# This file is being regenerated by this script.
FILENAME = 'db/structure.sql'
##
# Directories where migrations are stored
#
# The methods +hide_migrations+ and +unhide_migrations+ will rename
# these to disable/enable migrations.
MIGRATION_DIRS = %w[db/migrate db/post_migrate].freeze
##
# Directory where we store schema versions
#
# The remove_schema_migration_files removes files added in this
# directory when it runs.
SCHEMA_MIGRATIONS_DIR = 'db/schema_migrations/'
def initialize(options)
@rollback_testing = options.delete(:rollback_testing)
@init_schema_loading = options.delete(:init_schema_loading)
end
def execute
Dir.chdir(File.expand_path('..', __dir__)) do
# Note: `db:drop` must run prior to hiding migrations.
#
# Executing a Rails DB command e.g., `reset`, `drop`, etc. triggers running the initializers.
# During the initialization, the default values for `application_settings` need to be set.
# Depending on the presence of migrations, the default values are either faked or inserted.
#
# 1. If no migration is detected, all the necessary columns are in place from `db/structure.sql`.
# The default values can be inserted into `application_settings` table.
#
# 2. If a migration is detected, at least one column may be missing from `db/structure.sql`
# and needs to be added through the detected migration. In this case, the default values are faked.
# If not, an error would be raised e.g., "NoMethodError: undefined method `some_setting`"
#
# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/135085#note_1628210334 for more info.
#
load_tasks
drop_db
checkout_ref
checkout_clean_schema
hide_migrations
remove_schema_migration_files
stop_spring
setup_db
unhide_migrations
migrate
dump_schema
rollback if @rollback_testing
ensure
unhide_migrations
end
end
private
def load_tasks
require_relative '../config/environment'
Gitlab::Application.load_tasks
end
##
# Git checkout +CI_COMMIT_SHA+.
#
# When running from CI, checkout the clean commit,
# not the merged result.
def checkout_ref
return unless ci?
run %(git checkout #{source_ref})
run %q(git clean -f -- db)
end
##
# Checkout the clean schema from the target branch
def checkout_clean_schema
remote_checkout_clean_schema || local_checkout_clean_schema
end
##
# Get clean schema from remote servers
#
# This script might run in CI, using a shallow clone, so to checkout
# the file, fetch the target branch from the server.
def remote_checkout_clean_schema
return false unless project_url
return false unless target_project_url
run %(git remote add target_project #{target_project_url}.git)
run %(git fetch target_project #{target_branch}:#{target_branch})
local_checkout_clean_schema
end
##
# Git checkout the schema from target branch.
#
# Ask git to checkout the schema from the target branch and reset
# the file to unstage the changes.
def local_checkout_clean_schema
run %(git checkout #{merge_base} -- #{FILENAME})
run %(git reset -- #{FILENAME})
end
##
# Move migrations to where Rails will not find them.
#
# To reset the database to clean schema defined in +FILENAME+, move
# the migrations to a path where Rails will not find them, otherwise
# +db:reset+ would abort. Later when the migrations should be
# applied, use +unhide_migrations+ to bring them back.
def hide_migrations
MIGRATION_DIRS.each do |dir|
File.rename(dir, "#{dir}__")
end
end
##
# Undo the effect of +hide_migrations+.
#
# Place back the migrations which might be moved by
# +hide_migrations+.
def unhide_migrations
error = nil
MIGRATION_DIRS.each do |dir|
File.rename("#{dir}__", dir)
rescue Errno::ENOENT
nil
rescue StandardError => e
# Save error for later, but continue with other dirs first
error = e
end
raise error if error
end
##
# Remove files added to db/schema_migrations
#
# In order to properly reset the database and re-run migrations
# the schema migrations for new migrations must be removed.
def remove_schema_migration_files
(untracked_schema_migrations + committed_schema_migrations).each do |schema_migration|
FileUtils.rm(schema_migration)
end
end
##
# List of untracked schema migrations
#
# Get a list of schema migrations that are not tracked so we can remove them
def untracked_schema_migrations
git_command = "git ls-files --others --exclude-standard -- #{SCHEMA_MIGRATIONS_DIR}"
run(git_command).chomp.split("\n")
end
##
# List of untracked schema migrations
#
# Get a list of schema migrations that have been committed since the last
def committed_schema_migrations
git_command = "git diff --name-only --diff-filter=A #{merge_base} -- #{SCHEMA_MIGRATIONS_DIR}"
run(git_command).chomp.split("\n")
end
##
# Stop spring before modifying the database
def stop_spring
run %q(bin/spring stop)
end
##
# Run rake task to drop the database.
def drop_db
run_rake_task 'db:drop'
end
##
# Run rake task to setup the database.
def setup_db
run_rake_task(@init_schema_loading ? 'db:create' : 'db:setup')
end
##
# Run rake task to run migrations.
def migrate
run_rake_task 'db:migrate'
end
##
# Run rake task to dump schema.
def dump_schema
run_rake_task 'db:schema:dump'
end
##
# Run rake task to rollback migrations.
def rollback
(untracked_schema_migrations + committed_schema_migrations).sort.reverse_each do |filename|
version = filename[/\d+\Z/]
run %(bin/rails db:rollback:main db:rollback:ci RAILS_ENV=test VERSION=#{version})
end
end
##
# Run the given +cmd+.
#
# The command is colored green, and the output of the command is
# colored gray.
# When the command failed an exception is raised.
def run(cmd)
puts "\e[32m$ #{cmd}\e[37m"
stdout_str, stderr_str, status = Open3.capture3(cmd)
puts "#{stdout_str}#{stderr_str}\e[0m"
raise("Command failed: #{stderr_str}") unless status.success?
stdout_str
end
def run_rake_task(*tasks, env: {})
Array.wrap(tasks).each do |task|
env.each { |k, v| ENV[k.to_s] = v.to_s }
puts "\e[32m$ bin/rails #{task} RAILS_ENV=test #{env.map { |m| m.join('=') }.join(' ')}\e[37m"
Rake::Task[task].invoke
end
end
##
# Return the base commit between source and target branch.
def merge_base
@merge_base ||= run("git merge-base #{target_branch} #{source_ref}").chomp
end
##
# Return the name of the target branch
#
# Get source ref from CI environment variable, or read the +TARGET+
# environment+ variable, or default to +HEAD+.
def target_branch
ENV['CI_MERGE_REQUEST_TARGET_BRANCH_NAME'] || ENV['TARGET'] || ENV['CI_DEFAULT_BRANCH'] || 'master'
end
##
# Return the source ref
#
# Get source ref from CI environment variable, or default to +HEAD+.
def source_ref
ENV['CI_COMMIT_SHA'] || 'HEAD'
end
##
# Return the source project URL from CI environment variable.
def project_url
ENV['CI_PROJECT_URL']
end
##
# Return the target project URL from CI environment variable.
def target_project_url
ENV['CI_MERGE_REQUEST_PROJECT_URL']
end
##
# Return whether the script is running from CI
def ci?
ENV['CI']
end
end
if $PROGRAM_NAME == __FILE__
options = {}
OptionParser.new do |opts|
opts.on("-r", "--rollback-testing", String, "Enable rollback testing") do
options[:rollback_testing] = true
end
opts.on("-i", "--init-schema-loading", String,
"Enable clean migrations starting from the beginning - loading init_structure.sql") do
options[:init_schema_loading] = true
end
opts.on("-h", "--help", "Prints this help") do
puts opts
exit
end
end.parse!
SchemaRegenerator.new(options).execute
end
|