michael/pyfedi
1
0
Fork 0
mirror of https://codeberg.org/rimu/pyfedi synced 2025-01-23 19:36:56 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

faster migration #348

Browse source
This commit is contained in:
rimu 2024-11-15 10:54:21 +13:00
parent 05b0cdbbaf
commit bf9c849467
1 changed files with 50 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

97
migrations/versions/26138ecda7c3_unique_user_ap_profile_id.py
View file

@ -20,79 +20,66 @@ depends_on = None
def upgrade(): def upgrade():
# Find duplicate users by ap_profile_id # Find duplicate users by ap_profile_id
dupes_query = text(''' dupes_query = text('''
SELECT id FROM "user"
WHERE ap_profile_id IN (
SELECT ap_profile_id FROM "user" SELECT ap_profile_id FROM "user"
GROUP BY ap_profile_id GROUP BY ap_profile_id
HAVING COUNT(*) > 1 HAVING COUNT(*) > 1
)
''') ''')
conn = op.get_bind() conn = op.get_bind()
dupes = conn.execute(dupes_query).scalars() duplicate_profiles = conn.execute(dupes_query).scalars()
print('Cleaning up duplicate users, this may take a while...') print('Cleaning up duplicate users, this may take a while...')
for d in dupes:
user_query = text('SELECT id, ap_profile_id FROM "user" WHERE id = :id')
user = conn.execute(user_query, {"id": d}).first()
if not user: for profile_id in duplicate_profiles:
if profile_id is None:
continue continue
# Get all users with the same ap_profile_id, ordered by ID
# Find users with the same ap_profile_id
users_query = text(''' users_query = text('''
SELECT id FROM "user" SELECT id FROM "user"
WHERE ap_profile_id = :ap_profile_id WHERE ap_profile_id = :ap_profile_id
ORDER BY id ORDER BY id
''') ''')
users = conn.execute(users_query, {"ap_profile_id": user.ap_profile_id}).fetchall() users = conn.execute(users_query, {"ap_profile_id": profile_id}).fetchall()
first = True # Set the lowest ID as the new_id, and collect other IDs to update/delete
new_id = None new_id = users[0].id
old_ids = [user.id for user in users[1:]]
for u in users: if old_ids:
if first: # Update tables with batch IN clause
first = False conn.execute(text('UPDATE "post" SET user_id = :new_id WHERE user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
new_id = u.id conn.execute(text('DELETE FROM "post_vote" WHERE user_id IN :old_ids'), {"old_ids": tuple(old_ids)})
continue conn.execute(text('DELETE FROM "post_vote" WHERE author_id IN :old_ids'), {"old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "post_reply" SET user_id = :new_id WHERE user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('DELETE FROM "post_reply_vote" WHERE user_id IN :old_ids'), {"old_ids": tuple(old_ids)})
conn.execute(text('DELETE FROM "post_reply_vote" WHERE author_id IN :old_ids'), {"old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "notification" SET user_id = :new_id WHERE user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "notification" SET author_id = :new_id WHERE author_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "notification_subscription" SET user_id = :new_id WHERE user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('DELETE FROM "community_member" WHERE user_id IN :old_ids'), {"old_ids": tuple(old_ids)})
conn.execute(text('DELETE FROM "instance_role" WHERE user_id IN :old_ids'), {"old_ids": tuple(old_ids)})
conn.execute(text('DELETE FROM "mod_log" WHERE user_id IN :old_ids'), {"old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "chat_message" SET sender_id = :new_id WHERE sender_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "conversation" SET user_id = :new_id WHERE user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "chat_message" SET recipient_id = :new_id WHERE recipient_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('DELETE FROM "conversation_member" WHERE user_id IN :old_ids'), {"old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "community" SET user_id = :new_id WHERE user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "domain_block" SET user_id = :new_id WHERE user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "community_block" SET user_id = :new_id WHERE user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "user_follower" SET local_user_id = :new_id WHERE local_user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "user_follower" SET remote_user_id = :new_id WHERE remote_user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "community_ban" SET user_id = :new_id WHERE user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "user_note" SET target_id = :new_id WHERE target_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "user_block" SET blocked_id = :new_id WHERE blocked_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "community_join_request" SET user_id = :new_id WHERE user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "filter" SET user_id = :new_id WHERE user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "report" SET reporter_id = :new_id WHERE reporter_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "report" SET suspect_user_id = :new_id WHERE suspect_user_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
conn.execute(text('UPDATE "user_follow_request" SET follow_id = :new_id WHERE follow_id IN :old_ids'), {"new_id": new_id, "old_ids": tuple(old_ids)})
if new_id: # Delete the duplicate users
old_id = u.id conn.execute(text('DELETE FROM "user" WHERE id IN :old_ids'), {"old_ids": tuple(old_ids)})
# Update references in various tables # Finalize by dropping and recreating the unique index on ap_profile_id
conn.execute(text('UPDATE "post" SET user_id = :new_id WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('DELETE FROM "post_vote" WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('DELETE FROM "post_vote" WHERE author_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "post_reply" SET user_id = :new_id WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('DELETE FROM "post_reply_vote" WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('DELETE FROM "post_reply_vote" WHERE author_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "notification" SET user_id = :new_id WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "notification" SET author_id = :new_id WHERE author_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "notification_subscription" SET user_id = :new_id WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('DELETE FROM "community_member" WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('DELETE FROM "instance_role" WHERE user_id = :old_id'), {"old_id": old_id})
conn.execute(text('DELETE FROM "mod_log" WHERE user_id = :old_id'), {"old_id": old_id})
conn.execute(text('UPDATE "chat_message" SET sender_id = :new_id WHERE sender_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "conversation" SET user_id = :new_id WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "chat_message" SET recipient_id = :new_id WHERE recipient_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('DELETE FROM "conversation_member" WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "community" SET user_id = :new_id WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "domain_block" SET user_id = :new_id WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "community_block" SET user_id = :new_id WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "user_follower" SET local_user_id = :new_id WHERE local_user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "user_follower" SET remote_user_id = :new_id WHERE remote_user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "community_ban" SET user_id = :new_id WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "user_note" SET target_id = :new_id WHERE target_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "user_block" SET blocked_id = :new_id WHERE blocked_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "community_join_request" SET user_id = :new_id WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "filter" SET user_id = :new_id WHERE user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "report" SET reporter_id = :new_id WHERE reporter_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "report" SET suspect_user_id = :new_id WHERE suspect_user_id = :old_id'), {"new_id": new_id, "old_id": old_id})
conn.execute(text('UPDATE "user_follow_request" SET follow_id = :new_id WHERE follow_id = :old_id'), {"new_id": new_id, "old_id": old_id})
# Delete the duplicate user
conn.execute(text('DELETE FROM "user" WHERE id = :old_id'), {"old_id": old_id})
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('user', schema=None) as batch_op: with op.batch_alter_table('user', schema=None) as batch_op:
batch_op.drop_index('ix_user_ap_profile_id') batch_op.drop_index('ix_user_ap_profile_id')
batch_op.create_index(batch_op.f('ix_user_ap_profile_id'), ['ap_profile_id'], unique=True) batch_op.create_index(batch_op.f('ix_user_ap_profile_id'), ['ap_profile_id'], unique=True)