diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 523220d5..bbe61512 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -136,7 +136,7 @@ public static function getSupportedResources(): array * @throws AppwriteException */ #[Override] - public function report(array $resources = []): array + public function report(array $resources = [], array $resourceIds = []): array { if (empty($resources)) { $resources = $this->getSupportedResources(); diff --git a/src/Migration/Destinations/CSV.php b/src/Migration/Destinations/CSV.php index 12c29ad7..df37f092 100644 --- a/src/Migration/Destinations/CSV.php +++ b/src/Migration/Destinations/CSV.php @@ -65,7 +65,7 @@ public static function getSupportedResources(): array ]; } - public function report(array $resources = []): array + public function report(array $resources = [], array $resourceIds = []): array { return []; } diff --git a/src/Migration/Destinations/Local.php b/src/Migration/Destinations/Local.php index f195f1cb..4c17ab20 100644 --- a/src/Migration/Destinations/Local.php +++ b/src/Migration/Destinations/Local.php @@ -76,7 +76,7 @@ public static function getSupportedResources(): array /** * @throws \Exception */ - public function report(array $resources = []): array + public function report(array $resources = [], array $resourceIds = []): array { $report = []; diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 4216d40f..acc8f837 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -52,11 +52,7 @@ class Appwrite extends Source public const SOURCE_API = 'api'; public const SOURCE_DATABASE = 'database'; - // Debug logging for specific projects - public static array $debugProjects = [ - '67ec0369002bd8a96885' => 'SimpMusic#Maxrave', - '6838382d0014e002589c' => 'Fastwrite#DocuTrust', - ]; + private const DEFAULT_PAGE_LIMIT = 25; protected Client $client; @@ -163,12 +159,15 @@ public function getDatabasesBatchSize(): int /** * @param array $resources + * @param array> $resourceIds * @return array * * @throws \Exception */ - public function report(array $resources = []): array + public function report(array $resources = [], array $resourceIds = []): array { + $this->validateResourceIds($resourceIds); + $report = []; if (empty($resources)) { @@ -176,10 +175,10 @@ public function report(array $resources = []): array } try { - $this->reportAuth($resources, $report); - $this->reportDatabases($resources, $report); - $this->reportStorage($resources, $report); - $this->reportFunctions($resources, $report); + $this->reportAuth($resources, $report, $resourceIds); + $this->reportDatabases($resources, $report, $resourceIds); + $this->reportStorage($resources, $report, $resourceIds); + $this->reportFunctions($resources, $report, $resourceIds); $report['version'] = $this->call( 'GET', @@ -205,9 +204,10 @@ public function report(array $resources = []): array /** * @param array $resources * @param array $report + * @param array> $resourceIds * @throws AppwriteException */ - private function reportAuth(array $resources, array &$report): void + private function reportAuth(array $resources, array &$report, array $resourceIds = []): void { // check if we need to fetch teams! $needTeams = !empty(array_intersect( @@ -215,13 +215,16 @@ private function reportAuth(array $resources, array &$report): void $resources )); - $pageLimit = 25; $teams = ['total' => 0, 'teams' => []]; if (\in_array(Resource::TYPE_USER, $resources)) { - $report[Resource::TYPE_USER] = $this->users->list( - [Query::limit(1)] - )['total']; + $userQueries = $this->buildQueries( + resourceType: Resource::TYPE_USER, + resourceIds: $resourceIds, + limit: 1 + ); + $userList = $this->users->list($userQueries); + $report[Resource::TYPE_USER] = $userList['total']; } if ($needTeams) { @@ -230,12 +233,11 @@ private function reportAuth(array $resources, array &$report): void $lastTeam = null; while (true) { - $params = $lastTeam - // TODO: should we use offset here? - // this, realistically, shouldn't be too much ig - ? [Query::cursorAfter($lastTeam)] - : [Query::limit($pageLimit)]; - + $params = $this->buildQueries( + resourceType: Resource::TYPE_TEAM, + resourceIds: $resourceIds, + cursor: $lastTeam + ); $teamList = $this->teams->list($params); $totalTeams = $teamList['total']; @@ -244,13 +246,18 @@ private function reportAuth(array $resources, array &$report): void $allTeams = array_merge($allTeams, $currentTeams); $lastTeam = $currentTeams[count($currentTeams) - 1]['$id'] ?? null; - if (count($currentTeams) < $pageLimit) { + if (count($currentTeams) < self::DEFAULT_PAGE_LIMIT) { break; } } $teams = ['total' => $totalTeams, 'teams' => $allTeams]; } else { - $teamList = $this->teams->list([Query::limit(1)]); + $params = $this->buildQueries( + resourceType: Resource::TYPE_TEAM, + resourceIds: $resourceIds, + limit: 1 + ); + $teamList = $this->teams->list($params); $teams = ['total' => $teamList['total'], 'teams' => []]; } } @@ -274,27 +281,29 @@ private function reportAuth(array $resources, array &$report): void * @throws Exception * @throws AppwriteException */ - private function reportDatabases(array $resources, array &$report): void + private function reportDatabases(array $resources, array &$report, array $resourceIds = []): void { - $this->database->report($resources, $report); + $this->database->report($resources, $report, $resourceIds); } /** * @param array $resources * @param array $report + * @param array> $resourceIds * @throws AppwriteException */ - private function reportStorage(array $resources, array &$report): void + private function reportStorage(array $resources, array &$report, array $resourceIds = []): void { + if (\in_array(Resource::TYPE_BUCKET, $resources)) { - // just fetch one bucket for the `total` - $report[Resource::TYPE_BUCKET] = $this->storage->listBuckets([ - Query::limit(1) - ])['total']; + $bucketQueries = $this->buildQueries( + resourceType: Resource::TYPE_BUCKET, + resourceIds: $resourceIds, + limit: 1 + ); + $report[Resource::TYPE_BUCKET] = $this->storage->listBuckets($bucketQueries)['total']; } - $pageLimit = 25; - if (\in_array(Resource::TYPE_FILE, $resources)) { $report[Resource::TYPE_FILE] = 0; $report['size'] = 0; @@ -302,16 +311,17 @@ private function reportStorage(array $resources, array &$report): void $lastBucket = null; while (true) { - $currentBuckets = $this->storage->listBuckets( - $lastBucket - ? [Query::cursorAfter($lastBucket)] - : [Query::limit($pageLimit)] - )['buckets']; + $queries = $this->buildQueries( + resourceType: Resource::TYPE_BUCKET, + resourceIds: $resourceIds, + cursor: $lastBucket, + ); + $currentBuckets = $this->storage->listBuckets($queries)['buckets']; $buckets = array_merge($buckets, $currentBuckets); $lastBucket = $buckets[count($buckets) - 1]['$id'] ?? null; - if (count($currentBuckets) < $pageLimit) { + if (count($currentBuckets) < self::DEFAULT_PAGE_LIMIT) { break; } } @@ -319,11 +329,14 @@ private function reportStorage(array $resources, array &$report): void foreach ($buckets as $bucket) { $lastFile = null; while (true) { + $queries = [Query::limit(self::DEFAULT_PAGE_LIMIT)]; + if ($lastFile) { + $queries[] = Query::cursorAfter($lastFile); + } + $files = $this->storage->listFiles( $bucket['$id'], - $lastFile - ? [Query::cursorAfter($lastFile)] - : [Query::limit($pageLimit)] + $queries )['files']; $report[Resource::TYPE_FILE] += count($files); @@ -334,7 +347,7 @@ private function reportStorage(array $resources, array &$report): void $lastFile = $files[count($files) - 1]['$id'] ?? null; - if (count($files) < $pageLimit) { + if (count($files) < self::DEFAULT_PAGE_LIMIT) { break; } } @@ -344,9 +357,8 @@ private function reportStorage(array $resources, array &$report): void } } - private function reportFunctions(array $resources, array &$report): void + private function reportFunctions(array $resources, array &$report, array $resourceIds = []): void { - $pageLimit = 25; $needVarsOrDeployments = ( \in_array(Resource::TYPE_DEPLOYMENT, $resources) || \in_array(Resource::TYPE_ENVIRONMENT_VARIABLE, $resources) @@ -356,19 +368,23 @@ private function reportFunctions(array $resources, array &$report): void $totalFunctions = 0; if (!$needVarsOrDeployments && \in_array(Resource::TYPE_FUNCTION, $resources)) { - // Only function count needed, short-circuit - $funcList = $this->functions->list([Query::limit(1)]); - $report[Resource::TYPE_FUNCTION] = $funcList['total']; + $functionQueries = $this->buildQueries( + resourceType: Resource::TYPE_FUNCTION, + resourceIds: $resourceIds, + limit: 1 + ); + $report[Resource::TYPE_FUNCTION] = $this->functions->list($functionQueries)['total']; return; } if ($needVarsOrDeployments) { $lastFunction = null; while (true) { - $params = $lastFunction - ? [Query::cursorAfter($lastFunction)] - : [Query::limit($pageLimit)]; - + $params = $this->buildQueries( + resourceType: Resource::TYPE_FUNCTION, + resourceIds: $resourceIds, + cursor: $lastFunction, + ); $funcList = $this->functions->list($params); $totalFunctions = $funcList['total']; @@ -376,7 +392,7 @@ private function reportFunctions(array $resources, array &$report): void $functions = array_merge($functions, $currentFunctions); $lastFunction = $currentFunctions[count($currentFunctions) - 1]['$id'] ?? null; - if (count($currentFunctions) < $pageLimit) { + if (count($currentFunctions) < self::DEFAULT_PAGE_LIMIT) { break; } } @@ -1614,4 +1630,30 @@ private function exportDeploymentData(Func $func, array $deployment): void } } } + + /** + * Build queries with optional filtering by resource IDs + */ + private function buildQueries( + string $resourceType, + array $resourceIds, + ?string $cursor = null, + int $limit = self::DEFAULT_PAGE_LIMIT + ): array { + $queries = []; + + if (!empty($resourceIds[$resourceType])) { + $ids = (array) $resourceIds[$resourceType]; + + $queries[] = Query::equal('$id', $ids); + } + + if ($cursor) { + $queries[] = Query::cursorAfter($cursor); + } else { + $queries[] = Query::limit($limit); + } + + return $queries; + } } diff --git a/src/Migration/Sources/Appwrite/Reader.php b/src/Migration/Sources/Appwrite/Reader.php index b45f2bdc..314d4c33 100644 --- a/src/Migration/Sources/Appwrite/Reader.php +++ b/src/Migration/Sources/Appwrite/Reader.php @@ -16,9 +16,10 @@ interface Reader * * @param array $resources * @param array $report + * @param array> $resourceIds * @return mixed */ - public function report(array $resources, array &$report): mixed; + public function report(array $resources, array &$report, array $resourceIds = []): mixed; /** * List databases that match the given queries diff --git a/src/Migration/Sources/Appwrite/Reader/API.php b/src/Migration/Sources/Appwrite/Reader/API.php index da1a5022..388a0ce7 100644 --- a/src/Migration/Sources/Appwrite/Reader/API.php +++ b/src/Migration/Sources/Appwrite/Reader/API.php @@ -25,7 +25,7 @@ public function __construct( /** * @throws AppwriteException */ - public function report(array $resources, array &$report): mixed + public function report(array $resources, array &$report, array $resourceIds = []): mixed { $relevantResources = [ Resource::TYPE_DATABASE, @@ -45,7 +45,14 @@ public function report(array $resources, array &$report): mixed } } - $databasesResponse = $this->database->list(); + $databaseQueries = []; + if (!empty($resourceIds[Resource::TYPE_DATABASE])) { + $databaseIds = (array) $resourceIds[Resource::TYPE_DATABASE]; + + $databaseQueries[] = Query::equal('$id', $databaseIds); + } + + $databasesResponse = $this->database->list($databaseQueries); $databases = $databasesResponse['databases']; if (in_array(Resource::TYPE_DATABASE, $resources)) { diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index c43e246c..bad744ce 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -24,7 +24,7 @@ public function __construct(private readonly UtopiaDatabase $dbForProject) { } - public function report(array $resources, array &$report): mixed + public function report(array $resources, array &$report, array $resourceIds = []): mixed { $relevantResources = [ Resource::TYPE_DATABASE, @@ -44,8 +44,15 @@ public function report(array $resources, array &$report): mixed } } + $databaseQueries = []; + if (!empty($resourceIds[Resource::TYPE_DATABASE])) { + $databaseIds = (array) $resourceIds[Resource::TYPE_DATABASE]; + + $databaseQueries[] = Query::equal('$id', $databaseIds); + } + if (in_array(Resource::TYPE_DATABASE, $resources)) { - $report[Resource::TYPE_DATABASE] = $this->countResources('databases'); + $report[Resource::TYPE_DATABASE] = $this->countResources('databases', $databaseQueries); } if (count(array_intersect($resources, $relevantResources)) === 1 && @@ -54,7 +61,7 @@ public function report(array $resources, array &$report): mixed } $dbResources = []; - $databases = $this->listDatabases(); + $databases = $this->listDatabases($databaseQueries); // Process each database foreach ($databases as $database) { @@ -417,7 +424,6 @@ public function queryLimit(int $limit): Query * @param string $table * @param array $queries * @return int - * @throws DatabaseException */ private function countResources(string $table, array $queries = []): int { diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index a6ca0c2d..7aaeaa35 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -69,7 +69,7 @@ public static function getSupportedResources(): array /** * @throws \Exception */ - public function report(array $resources = []): array + public function report(array $resources = [], array $resourceIds = []): array { $report = []; diff --git a/src/Migration/Sources/Firebase.php b/src/Migration/Sources/Firebase.php index 917cc0bc..12117d6a 100644 --- a/src/Migration/Sources/Firebase.php +++ b/src/Migration/Sources/Firebase.php @@ -141,7 +141,7 @@ public static function getSupportedResources(): array ]; } - public function report(array $resources = []): array + public function report(array $resources = [], array $resourceIds = []): array { // Check our service account is valid if (! isset($this->serviceAccount['project_id'])) { diff --git a/src/Migration/Sources/NHost.php b/src/Migration/Sources/NHost.php index aa29db3b..f65e7005 100644 --- a/src/Migration/Sources/NHost.php +++ b/src/Migration/Sources/NHost.php @@ -100,7 +100,7 @@ public static function getSupportedResources(): array ]; } - public function report(array $resources = []): array + public function report(array $resources = [], array $resourceIds = []): array { $report = []; diff --git a/src/Migration/Sources/Supabase.php b/src/Migration/Sources/Supabase.php index 489f5211..05006e19 100644 --- a/src/Migration/Sources/Supabase.php +++ b/src/Migration/Sources/Supabase.php @@ -229,7 +229,7 @@ public function __construct(string $endpoint, string $key, string $host, string } } - public function report(array $resources = []): array + public function report(array $resources = [], array $resourceIds = []): array { $report = []; diff --git a/src/Migration/Target.php b/src/Migration/Target.php index 633a85c7..e1eee959 100644 --- a/src/Migration/Target.php +++ b/src/Migration/Target.php @@ -63,9 +63,12 @@ abstract public function run(array $resources, callable $callback, string $rootR * If any issues are found then an exception should be thrown with an error message. * * @param array $resources Resources to report + * @param array> $resourceIds Map of resource type to IDs. Only top-level resources supported. * @return array + * + * @throws \Exception if resourceIds contains non-top-level resource types */ - abstract public function report(array $resources = []): array; + abstract public function report(array $resources = [], array $resourceIds = []): array; /** * Make an API call @@ -183,6 +186,21 @@ protected function flatten(array $data, string $prefix = ''): array return $output; } + /** + * Validate that resourceIds only contains top-level resources + */ + protected function validateResourceIds(array $resourceIds): void + { + foreach (array_keys($resourceIds) as $resourceType) { + if (!in_array($resourceType, Transfer::ROOT_RESOURCES)) { + throw new \Exception( + 'Invalid resource type in resourceIds: ' . $resourceType . '. ' . + 'Only top-level resources are supported: ' . implode(', ', Transfer::ROOT_RESOURCES) + ); + } + } + } + /** * Get Errors * diff --git a/tests/Migration/Unit/Adapters/MockDestination.php b/tests/Migration/Unit/Adapters/MockDestination.php index d9d3bb54..7c9806c6 100644 --- a/tests/Migration/Unit/Adapters/MockDestination.php +++ b/tests/Migration/Unit/Adapters/MockDestination.php @@ -85,7 +85,7 @@ public function import(array $resources, callable $callback): void $callback($resources); } - public function report(array $resources = []): array + public function report(array $resources = [], array $resourceIds = []): array { return []; } diff --git a/tests/Migration/Unit/Adapters/MockSource.php b/tests/Migration/Unit/Adapters/MockSource.php index ef599599..41d352e3 100644 --- a/tests/Migration/Unit/Adapters/MockSource.php +++ b/tests/Migration/Unit/Adapters/MockSource.php @@ -85,7 +85,7 @@ public static function getSupportedResources(): array ]; } - public function report(array $resources = []): array + public function report(array $resources = [], array $resourceIds = []): array { return []; }