· 4 years ago · Jul 20, 2021, 09:36 PM
1import * as cmd from '../cmd';
2import config from '../config';
3import * as dynamo from '../connectors/dynamo';
4import * as route53 from '../connectors/route53';
5import * as queueConstants from '../connectors/queue/constants';
6import {RelocationResource} from '../../admin/server/resources/relocation_res';
7import {getPublicApiDomainName} from '../util/dns';
8import CompanyResource from '../resources/company_res';
9import TeammateResource from '../resources/teammate_res';
10import {getCellByRegionAndShardKey} from '../util/cells';
11import {DynamoGlobalCompany} from '../util/cells/types';
12import {NO_CELL} from '../util/cells/constants';
13import {RelocationAction} from './relocation_action';
14import {ddbBatchSize} from './constants';
15import {clearCompanyCacheInAllCells, clearTeammatesCacheInAllCells} from './clear_caches';
16import {blockAllProxies, checkProxies, unblockAllProxies} from './proxy_control';
17import {uncacheAsync} from '../domain/reqcache';
18import {StorageRelocation} from './storages/types';
19import {MysqlStorageRelocation, ElasticsearchStorageRelocation, RedisStorageRelocation} from './storages';
20
21const publicApiRoute53 = route53.zone(config('public_api_v2.hosted_zone_id'));
22
23const globalCompanies = dynamo.table(`front-app-${config('env_namespace')}-companies`);
24const globalTeammates = dynamo.table(`front-app-${config('env_namespace')}-teammates`);
25
26export class Relocate extends RelocationAction {
27 unblockProxiesOnError = false;
28 resume?: boolean;
29 storageRelocations: StorageRelocation[];
30
31 constructor(relocation: RelocationResource, {resume}: {resume?: boolean} = {}) {
32 super('relocate', relocation);
33
34 this.resume = resume;
35 // TODO: add ddb
36 this.storageRelocations = [MysqlStorageRelocation, ElasticsearchStorageRelocation, RedisStorageRelocation].map(
37 (SR) => new SR(relocation, {resume})
38 );
39 }
40
41 private async checkCanRelocate(globalCompany: DynamoGlobalCompany) {
42 await this.logStep('Verifying that relocation is allowed');
43 if (globalCompany.region !== this.sourceRegion || globalCompany.shard_key !== this.sourceShardKey) {
44 throw new Error(`Relocation forbidden for company ${this.companyId}`);
45 }
46
47 // This is necessary for ES relocation as indices are prefixed with their cell and the correct configs are loaded as a result
48 // Supporting cross region no_cell to no_cell relocation complicates the config system and we should prefer relocating to a cell
49 if (this.destCell === NO_CELL) {
50 throw new Error('Relocation to no_cell is not allowed. Relocate to a cell in the destination region');
51 }
52
53 if (!this.sameRegion) {
54 await this.logStep('Verifying that cross-region relocation is allowed');
55 const company = await CompanyResource.fetch(this.companyId);
56
57 if (!company.isCrossRegionAllowed()) {
58 throw new Error(`Cross region relocation is forbidden for company ${this.companyId}`);
59 }
60 }
61
62 for (const storageRelocation of this.storageRelocations) {
63 await storageRelocation.checkCanRelocate?.();
64 }
65 }
66
67 private async sendGlobalEvents() {
68 await this.logStep('Clearing global resources cache and restarting channels');
69
70 // TODO: No reason to send this to the local region for in-region
71 // relocations, but there's not a good way to exclude a region
72 // from a GLOBAL message at this point.
73 const events = ['all_regions_clear_global_cache', 'new_region_restart_channels'];
74 const data = {
75 company_id: this.companyId,
76 _options: {replications: queueConstants.Replication.GLOBAL},
77 };
78
79 for (const event of events) {
80 await cmd.queueNowAsync(event, data);
81 }
82
83 if (!this.sameRegion) {
84 await this.logStep('Clearing resources cache');
85 // TODO: For cross-region relocations we should have another way
86 // to clear the caches without relying on a queue event.
87 await cmd.queueNowAsync('new_region_clear_cache', data);
88 }
89 }
90
91 private async updatePublicApiDns() {
92 const cell = getCellByRegionAndShardKey(this.destRegion, this.destShardKey);
93 // Refetch company as subdomain might have changed in the meantime
94 const {subdomain} = await globalCompanies.fetchAsync(this.companyId);
95 const name = getPublicApiDomainName(subdomain);
96 const target = getPublicApiDomainName(cell);
97 await this.logStep(`Update public API DNS record from ${name} to ${target}`);
98 await publicApiRoute53.upsertAlias(name, target);
99 }
100
101 private async cutover({slug}: DynamoGlobalCompany) {
102 await this.logStep('Updating company region and shard key in DynamoDB');
103 this.relocation.shouldPageOnError = true;
104 this.unblockProxiesOnError = false;
105 const dynamoUpdateBlob = {
106 id: this.companyId,
107 region: this.destRegion,
108 shard_key: this.destShardKey,
109 // TODO: Improve dynamo connector to support removing fields on update
110 relocationMeta: {},
111 };
112 await globalCompanies.updateAsync(dynamoUpdateBlob);
113
114 await this.logStep('Retrieving teammates');
115 const teammates = await TeammateResource.listGlobalByCompanyIdAsync(this.companyId);
116
117 if (!this.sameRegion) {
118 await this.logStep('Updating teammates region in DynamoDB');
119 const dynamoBatchWriteArray = teammates.map((teammate: any) => ({
120 ...teammate,
121 region: this.destRegion,
122 }));
123
124 while (dynamoBatchWriteArray.length > 0) {
125 const batch = dynamoBatchWriteArray.splice(0, ddbBatchSize);
126 await globalTeammates.batchWriteAsync(batch);
127 }
128 }
129
130 await this.logStep('Clearing local resources cache');
131 await clearCompanyCacheInAllCells(this.companyId, slug);
132 await clearTeammatesCacheInAllCells(this.companyId, teammates);
133 // Uncache memory-cached company so that it's actually fetched from the DB or the cache afterwards
134 await uncacheAsync(CompanyResource.className, this.companyId);
135
136 await this.sendGlobalEvents();
137 }
138
139 async run() {
140 try {
141 const globalCompany = await globalCompanies.fetchAsync(this.companyId);
142
143 if (!this.resume) {
144 await this.checkCanRelocate(globalCompany);
145 await this.logStep('Putting record on DynamoDB');
146 await this.relocation.setRelocationStarted();
147 }
148
149 for (const storageRelocation of this.storageRelocations) {
150 await storageRelocation.start?.();
151 }
152
153 for (const storageRelocation of this.storageRelocations) {
154 await storageRelocation.wait?.();
155 }
156
157 if (!this.sameCell) {
158 await this.logStep('Blocking company in proxies');
159 this.unblockProxiesOnError = true;
160 const blockProxies = await blockAllProxies(this.sourceCell, this.companyId);
161
162 for (const storageRelocation of this.storageRelocations) {
163 await storageRelocation.preCutover?.();
164 }
165
166 for (const storageRelocation of this.storageRelocations) {
167 await storageRelocation.waitPointInTime?.();
168 }
169
170 await this.logStep('Checking proxy block is still valid');
171 await checkProxies(this.sourceCell, blockProxies);
172 }
173
174 await this.cutover(globalCompany);
175
176 if (!this.sameRegion || !this.sameCell) {
177 await this.updatePublicApiDns();
178 }
179
180 for (const storageRelocation of this.storageRelocations) {
181 await storageRelocation.stop?.();
182 }
183
184 await this.logStep('Removing record from DynamoDB');
185 await this.relocation.setRelocationFinished();
186 } catch (err) {
187 if (this.unblockProxiesOnError) {
188 try {
189 await unblockAllProxies(this.sourceCell, this.companyId);
190 } catch (unblockErr) {
191 this.logError('Failed to unblock proxies after error', unblockErr);
192 }
193 }
194
195 throw err;
196 }
197 }
198}
199