|
|
|
@ -84,7 +84,7 @@ module Mastodon
|
|
|
|
|
|
|
|
|
|
|
|
BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job
|
|
|
|
BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job
|
|
|
|
BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time
|
|
|
|
BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time
|
|
|
|
|
|
|
|
|
|
|
|
# Gets an estimated number of rows for a table
|
|
|
|
# Gets an estimated number of rows for a table
|
|
|
|
def estimate_rows_in_table(table_name)
|
|
|
|
def estimate_rows_in_table(table_name)
|
|
|
|
exec_query('SELECT reltuples FROM pg_class WHERE relname = ' +
|
|
|
|
exec_query('SELECT reltuples FROM pg_class WHERE relname = ' +
|
|
|
|
@ -313,14 +313,14 @@ module Mastodon
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
table = Arel::Table.new(table_name)
|
|
|
|
table = Arel::Table.new(table_name)
|
|
|
|
|
|
|
|
|
|
|
|
total = estimate_rows_in_table(table_name).to_i
|
|
|
|
total = estimate_rows_in_table(table_name).to_i
|
|
|
|
if total == 0
|
|
|
|
if total == 0
|
|
|
|
count_arel = table.project(Arel.star.count.as('count'))
|
|
|
|
count_arel = table.project(Arel.star.count.as('count'))
|
|
|
|
count_arel = yield table, count_arel if block_given?
|
|
|
|
count_arel = yield table, count_arel if block_given?
|
|
|
|
|
|
|
|
|
|
|
|
total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i
|
|
|
|
total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i
|
|
|
|
|
|
|
|
|
|
|
|
return if total == 0
|
|
|
|
return if total == 0
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
@ -339,7 +339,7 @@ module Mastodon
|
|
|
|
# In case there are no rows but we didn't catch it in the estimated size:
|
|
|
|
# In case there are no rows but we didn't catch it in the estimated size:
|
|
|
|
return unless first_row
|
|
|
|
return unless first_row
|
|
|
|
start_id = first_row['id'].to_i
|
|
|
|
start_id = first_row['id'].to_i
|
|
|
|
|
|
|
|
|
|
|
|
say "Migrating #{table_name}.#{column} (~#{total.to_i} rows)"
|
|
|
|
say "Migrating #{table_name}.#{column} (~#{total.to_i} rows)"
|
|
|
|
|
|
|
|
|
|
|
|
started_time = Time.now
|
|
|
|
started_time = Time.now
|
|
|
|
@ -347,7 +347,7 @@ module Mastodon
|
|
|
|
migrated = 0
|
|
|
|
migrated = 0
|
|
|
|
loop do
|
|
|
|
loop do
|
|
|
|
stop_row = nil
|
|
|
|
stop_row = nil
|
|
|
|
|
|
|
|
|
|
|
|
suppress_messages do
|
|
|
|
suppress_messages do
|
|
|
|
stop_arel = table.project(table[:id])
|
|
|
|
stop_arel = table.project(table[:id])
|
|
|
|
.where(table[:id].gteq(start_id))
|
|
|
|
.where(table[:id].gteq(start_id))
|
|
|
|
@ -373,29 +373,29 @@ module Mastodon
|
|
|
|
|
|
|
|
|
|
|
|
execute(update_arel.to_sql)
|
|
|
|
execute(update_arel.to_sql)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
migrated += batch_size
|
|
|
|
migrated += batch_size
|
|
|
|
if Time.now - last_time > 1
|
|
|
|
if Time.now - last_time > 1
|
|
|
|
status = "Migrated #{migrated} rows"
|
|
|
|
status = "Migrated #{migrated} rows"
|
|
|
|
|
|
|
|
|
|
|
|
percentage = 100.0 * migrated / total
|
|
|
|
percentage = 100.0 * migrated / total
|
|
|
|
status += " (~#{sprintf('%.2f', percentage)}%, "
|
|
|
|
status += " (~#{sprintf('%.2f', percentage)}%, "
|
|
|
|
|
|
|
|
|
|
|
|
remaining_time = (100.0 - percentage) * (Time.now - started_time) / percentage
|
|
|
|
remaining_time = (100.0 - percentage) * (Time.now - started_time) / percentage
|
|
|
|
|
|
|
|
|
|
|
|
status += "#{(remaining_time / 60).to_i}:"
|
|
|
|
status += "#{(remaining_time / 60).to_i}:"
|
|
|
|
status += sprintf('%02d', remaining_time.to_i % 60)
|
|
|
|
status += sprintf('%02d', remaining_time.to_i % 60)
|
|
|
|
status += ' remaining, '
|
|
|
|
status += ' remaining, '
|
|
|
|
|
|
|
|
|
|
|
|
# Tell users not to interrupt if we're almost done.
|
|
|
|
# Tell users not to interrupt if we're almost done.
|
|
|
|
if remaining_time > 10
|
|
|
|
if remaining_time > 10
|
|
|
|
status += 'safe to interrupt'
|
|
|
|
status += 'safe to interrupt'
|
|
|
|
else
|
|
|
|
else
|
|
|
|
status += 'DO NOT interrupt'
|
|
|
|
status += 'DO NOT interrupt'
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
status += ')'
|
|
|
|
status += ')'
|
|
|
|
|
|
|
|
|
|
|
|
say status, true
|
|
|
|
say status, true
|
|
|
|
last_time = Time.now
|
|
|
|
last_time = Time.now
|
|
|
|
end
|
|
|
|
end
|
|
|
|
@ -483,7 +483,7 @@ module Mastodon
|
|
|
|
|
|
|
|
|
|
|
|
check_trigger_permissions!(table)
|
|
|
|
check_trigger_permissions!(table)
|
|
|
|
trigger_name = rename_trigger_name(table, old, new)
|
|
|
|
trigger_name = rename_trigger_name(table, old, new)
|
|
|
|
|
|
|
|
|
|
|
|
# If we were in the middle of update_column_in_batches, we should remove
|
|
|
|
# If we were in the middle of update_column_in_batches, we should remove
|
|
|
|
# the old column and start over, as we have no idea where we were.
|
|
|
|
# the old column and start over, as we have no idea where we were.
|
|
|
|
if column_for(table, new)
|
|
|
|
if column_for(table, new)
|
|
|
|
@ -492,7 +492,7 @@ module Mastodon
|
|
|
|
else
|
|
|
|
else
|
|
|
|
remove_rename_triggers_for_mysql(trigger_name)
|
|
|
|
remove_rename_triggers_for_mysql(trigger_name)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
remove_column(table, new)
|
|
|
|
remove_column(table, new)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
@ -546,12 +546,12 @@ module Mastodon
|
|
|
|
temp_column = rename_column_name(column)
|
|
|
|
temp_column = rename_column_name(column)
|
|
|
|
|
|
|
|
|
|
|
|
rename_column_concurrently(table, column, temp_column, type: new_type)
|
|
|
|
rename_column_concurrently(table, column, temp_column, type: new_type)
|
|
|
|
|
|
|
|
|
|
|
|
# Primary keys don't necessarily have an associated index.
|
|
|
|
# Primary keys don't necessarily have an associated index.
|
|
|
|
if ActiveRecord::Base.get_primary_key(table) == column.to_s
|
|
|
|
if ActiveRecord::Base.get_primary_key(table) == column.to_s
|
|
|
|
old_pk_index_name = "index_#{table}_on_#{column}"
|
|
|
|
old_pk_index_name = "index_#{table}_on_#{column}"
|
|
|
|
new_pk_index_name = "index_#{table}_on_#{column}_cm"
|
|
|
|
new_pk_index_name = "index_#{table}_on_#{column}_cm"
|
|
|
|
|
|
|
|
|
|
|
|
unless indexes_for(table, column).find{|i| i.name == old_pk_index_name}
|
|
|
|
unless indexes_for(table, column).find{|i| i.name == old_pk_index_name}
|
|
|
|
add_concurrent_index(table, [temp_column], {
|
|
|
|
add_concurrent_index(table, [temp_column], {
|
|
|
|
unique: true,
|
|
|
|
unique: true,
|
|
|
|
@ -572,14 +572,14 @@ module Mastodon
|
|
|
|
# Wait for the indices to be built
|
|
|
|
# Wait for the indices to be built
|
|
|
|
indexes_for(table, column).each do |index|
|
|
|
|
indexes_for(table, column).each do |index|
|
|
|
|
expected_name = index.name + '_cm'
|
|
|
|
expected_name = index.name + '_cm'
|
|
|
|
|
|
|
|
|
|
|
|
puts "Waiting for index #{expected_name}"
|
|
|
|
puts "Waiting for index #{expected_name}"
|
|
|
|
sleep 1 until indexes_for(table, temp_column).find {|i| i.name == expected_name }
|
|
|
|
sleep 1 until indexes_for(table, temp_column).find {|i| i.name == expected_name }
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
was_primary = (ActiveRecord::Base.get_primary_key(table) == column.to_s)
|
|
|
|
was_primary = (ActiveRecord::Base.get_primary_key(table) == column.to_s)
|
|
|
|
old_default_fn = column_for(table, column).default_function
|
|
|
|
old_default_fn = column_for(table, column).default_function
|
|
|
|
|
|
|
|
|
|
|
|
old_fks = []
|
|
|
|
old_fks = []
|
|
|
|
if was_primary
|
|
|
|
if was_primary
|
|
|
|
# Get any foreign keys pointing at this column we need to recreate, and
|
|
|
|
# Get any foreign keys pointing at this column we need to recreate, and
|
|
|
|
@ -613,7 +613,7 @@ module Mastodon
|
|
|
|
target_col: temp_column,
|
|
|
|
target_col: temp_column,
|
|
|
|
on_delete: extract_foreign_key_action(old_fk['on_delete'])
|
|
|
|
on_delete: extract_foreign_key_action(old_fk['on_delete'])
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
remove_foreign_key(old_fk['src_table'], name: old_fk['name'])
|
|
|
|
remove_foreign_key(old_fk['src_table'], name: old_fk['name'])
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
@ -629,15 +629,15 @@ module Mastodon
|
|
|
|
transaction do
|
|
|
|
transaction do
|
|
|
|
# This has to be performed in a transaction as otherwise we might have
|
|
|
|
# This has to be performed in a transaction as otherwise we might have
|
|
|
|
# inconsistent data.
|
|
|
|
# inconsistent data.
|
|
|
|
|
|
|
|
|
|
|
|
cleanup_concurrent_column_rename(table, column, temp_column)
|
|
|
|
cleanup_concurrent_column_rename(table, column, temp_column)
|
|
|
|
rename_column(table, temp_column, column)
|
|
|
|
rename_column(table, temp_column, column)
|
|
|
|
|
|
|
|
|
|
|
|
# If there was an old default function, we didn't copy it. Do that now
|
|
|
|
# If there was an old default function, we didn't copy it. Do that now
|
|
|
|
# in the transaction, so we don't miss anything.
|
|
|
|
# in the transaction, so we don't miss anything.
|
|
|
|
change_column_default(table, column, -> { old_default_fn }) if old_default_fn
|
|
|
|
change_column_default(table, column, -> { old_default_fn }) if old_default_fn
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
# Rename any indices back to what they should be.
|
|
|
|
# Rename any indices back to what they should be.
|
|
|
|
indexes_for(table, column).each do |index|
|
|
|
|
indexes_for(table, column).each do |index|
|
|
|
|
next unless index.name.end_with?('_cm')
|
|
|
|
next unless index.name.end_with?('_cm')
|
|
|
|
@ -645,7 +645,7 @@ module Mastodon
|
|
|
|
real_index_name = index.name.sub(/_cm$/, '')
|
|
|
|
real_index_name = index.name.sub(/_cm$/, '')
|
|
|
|
rename_index(table, index.name, real_index_name)
|
|
|
|
rename_index(table, index.name, real_index_name)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
# Rename any foreign keys back to names based on the real column.
|
|
|
|
# Rename any foreign keys back to names based on the real column.
|
|
|
|
foreign_keys_for(table, column).each do |fk|
|
|
|
|
foreign_keys_for(table, column).each do |fk|
|
|
|
|
old_fk_name = concurrent_foreign_key_name(fk.from_table, temp_column, 'id')
|
|
|
|
old_fk_name = concurrent_foreign_key_name(fk.from_table, temp_column, 'id')
|
|
|
|
@ -653,7 +653,7 @@ module Mastodon
|
|
|
|
execute("ALTER TABLE #{fk.from_table} RENAME CONSTRAINT " +
|
|
|
|
execute("ALTER TABLE #{fk.from_table} RENAME CONSTRAINT " +
|
|
|
|
"#{old_fk_name} TO #{new_fk_name}")
|
|
|
|
"#{old_fk_name} TO #{new_fk_name}")
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
# Rename any foreign keys from other tables to names based on the real
|
|
|
|
# Rename any foreign keys from other tables to names based on the real
|
|
|
|
# column.
|
|
|
|
# column.
|
|
|
|
old_fks.each do |old_fk|
|
|
|
|
old_fks.each do |old_fk|
|
|
|
|
@ -664,7 +664,7 @@ module Mastodon
|
|
|
|
execute("ALTER TABLE #{old_fk['src_table']} RENAME CONSTRAINT " +
|
|
|
|
execute("ALTER TABLE #{old_fk['src_table']} RENAME CONSTRAINT " +
|
|
|
|
"#{old_fk_name} TO #{new_fk_name}")
|
|
|
|
"#{old_fk_name} TO #{new_fk_name}")
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
# If the old column was a primary key, mark the new one as a primary key.
|
|
|
|
# If the old column was a primary key, mark the new one as a primary key.
|
|
|
|
if was_primary
|
|
|
|
if was_primary
|
|
|
|
execute("ALTER TABLE #{table} ADD PRIMARY KEY USING INDEX " +
|
|
|
|
execute("ALTER TABLE #{table} ADD PRIMARY KEY USING INDEX " +
|
|
|
|
@ -791,7 +791,7 @@ module Mastodon
|
|
|
|
# This is necessary as we can't properly rename indexes such as
|
|
|
|
# This is necessary as we can't properly rename indexes such as
|
|
|
|
# "ci_taggings_idx".
|
|
|
|
# "ci_taggings_idx".
|
|
|
|
name = index.name + '_cm'
|
|
|
|
name = index.name + '_cm'
|
|
|
|
|
|
|
|
|
|
|
|
# If the order contained the old column, map it to the new one.
|
|
|
|
# If the order contained the old column, map it to the new one.
|
|
|
|
order = index.orders
|
|
|
|
order = index.orders
|
|
|
|
if order.key?(old)
|
|
|
|
if order.key?(old)
|
|
|
|
|
David Yip
7 years ago