diff --git a/batch/batches/Storage/Engine/KFileTransferExportEngine.php b/batch/batches/Storage/Engine/KFileTransferExportEngine.php index 9f488bbf0a0..f416fffbdd4 100644 --- a/batch/batches/Storage/Engine/KFileTransferExportEngine.php +++ b/batch/batches/Storage/Engine/KFileTransferExportEngine.php @@ -27,23 +27,24 @@ function __construct($data, $jobSubType) { function export() { $srcTempFile = null; - - $srcFile = kFile::realPath($this->srcFile); - $realPathRemote = strpos($srcFile, "http") !== false && kFile::checkFileExists($this->srcFile); - if($realPathRemote || !KBatchBase::pollingFileExists($this->srcFile)) + + list($isRemote, $remoteUrl, $isDir) = kFile::resolveFilePath($this->srcFile); + $realPathRemote = $isRemote && kFile::checkFileExists($this->srcFile); + if( $realPathRemote || !KBatchBase::pollingFileExists($this->srcFile)) { - $externalUrl = $realPathRemote ? $srcFile : $this->data->externalUrl; + $externalUrl = $isRemote ? $remoteUrl : $this->data->externalUrl; if($externalUrl == null) { throw new kTemporaryException("Source file {$this->srcFile} does not exist, and no external URL provided"); } - - $srcTempFile = $this->getAssetFile($externalUrl, $this->data->flavorAssetId); + + $srcTempFile = kFile::fetchRemoteToLocal($this->srcFile, $externalUrl, $isDir, + sys_get_temp_dir() . "/imports/", $this->data->flavorAssetId . "_" . basename($this->srcFile)); + if(!$srcTempFile) { throw new kTemporaryException("Source file {$this->srcFile} does not exist"); } - $this->srcFile = $srcTempFile; } @@ -93,6 +94,7 @@ function export() else if (kFile::isDir($this->srcFile)) { $filesPaths = kFile::dirList($this->srcFile); + $destDir = $this->destFile; foreach ($filesPaths as $filePath) { @@ -229,56 +231,29 @@ protected function addS3FieldsToStorageData($storageExportData, $externalStorage return $storageExportData; } - protected function getAssetFile($externalUrl, $uniqueDownloadId = null) + protected function unlinkFileIfNeeded($tempFile) { - // Needed arguments - if($externalUrl === null) - { - KalturaLog::info("No external url provided,"); - return null; - } - - // Create the temporary file path - $tempDirectoryPath = sys_get_temp_dir(); - if (!kFile::isDir($tempDirectoryPath)) - { - kFile::fullMkfileDir($tempDirectoryPath, 0700, true); - } - - if(!$uniqueDownloadId) + if (!$tempFile) { - $uniqueDownloadId = uniqid(rand(),true); + return; } - - $filePath = $tempDirectoryPath . "/asset_$uniqueDownloadId"; - // Retrieve the file - $res = null; - try + if (kFile::isDir($tempFile)) { - $stat = KCurlWrapper::getDataFromFile($externalUrl, $filePath, null, true); - - if ($stat) + $dir = dir($tempFile); + if (!$dir) { - $res = $filePath; - KalturaLog::info("Succeeded to retrieve asset content for assetId: [$assetId]"); + return null; } - else + while ($dirFile = $dir->read()) { - KalturaLog::info("Failed to retrieve asset content for assetId: [$assetId]"); + if ($dirFile != "." && $dirFile != "..") + { + unlink($tempFile . DIRECTORY_SEPARATOR . $dirFile); + } } } - catch(Exception $e) - { - KalturaLog::info("Can't serve asset id [$assetId] from [$externalUrl] " . $e->getMessage()); - } - - return $res; - } - - protected function unlinkFileIfNeeded($tempFile) - { - if($tempFile) + else { unlink($tempFile); } diff --git a/infra/storage/kFile.class.php b/infra/storage/kFile.class.php index 4cd15bd64a5..35d823a2c2d 100644 --- a/infra/storage/kFile.class.php +++ b/infra/storage/kFile.class.php @@ -61,8 +61,15 @@ public static function listDir($path, $pathPrefix = '') } // TODO - implement recursion - static public function dirList($directory, $return_directory_as_prefix = true, $should_recurse = false) + static public function dirList($directory, $return_directory_as_prefix = true) { + if (kFile::isSharedPath($directory)) + { + $sharedFsMgr = kSharedFileSystemMgr::getInstanceFromPath($directory); + $pathPrefix = $return_directory_as_prefix ? $directory . "/" : ''; + return $sharedFsMgr->listFiles($directory . "/" , $pathPrefix, false, true); + } + // create an array to hold directory list $results = array(); @@ -74,10 +81,9 @@ static public function dirList($directory, $return_directory_as_prefix = true, $ // keep going until all files in directory have been read while($file = readdir($handler)) { - // if $file isn't this directory or its parent, // add it to the results array - if($file != '.' && $file != '..') + if($file != '.' && $file != '..' ) { $results[] = ($return_directory_as_prefix ? $directory . "/" : "") . $file; } diff --git a/infra/storage/shared_file_system_managers/kNfsSharedFileSystemMgr.php b/infra/storage/shared_file_system_managers/kNfsSharedFileSystemMgr.php index fc47e329898..267904c1494 100644 --- a/infra/storage/shared_file_system_managers/kNfsSharedFileSystemMgr.php +++ b/infra/storage/shared_file_system_managers/kNfsSharedFileSystemMgr.php @@ -221,7 +221,7 @@ protected function doGetUploadMaxSize() return 2000000000; } - protected function doListFiles($filePath, $pathPrefix = '') + protected function doListFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false) { $fileList = array(); $path = str_ireplace(DIRECTORY_SEPARATOR, '/', $filePath); @@ -238,12 +238,15 @@ protected function doListFiles($filePath, $pathPrefix = '') if (is_dir($fullPath)) { $tmpPrefix = $tmpPrefix.'/'; - $fileList[] = array($tmpPrefix, 'dir', self::fileSize($fullPath)); - $fileList = array_merge($fileList, self::listDir($fullPath, $tmpPrefix)); + $fileList[] = $fileNamesOnly ? $tmpPrefix : array($tmpPrefix, 'dir', self::fileSize($fullPath)); + if ($recursive) + { + $fileList = array_merge($fileList, self::doListFiles($fullPath, $tmpPrefix)); + } } else { - $fileList[] = array($tmpPrefix, 'file', self::fileSize($fullPath)); + $fileList[] = $fileNamesOnly ? $tmpPrefix : array($tmpPrefix, 'file', self::fileSize($fullPath)); } } } diff --git a/infra/storage/shared_file_system_managers/kS3SharedFileSystemMgr.php b/infra/storage/shared_file_system_managers/kS3SharedFileSystemMgr.php index 85e4d12ac2b..db022a8e56f 100644 --- a/infra/storage/shared_file_system_managers/kS3SharedFileSystemMgr.php +++ b/infra/storage/shared_file_system_managers/kS3SharedFileSystemMgr.php @@ -578,19 +578,19 @@ public function completeMultiPartUpload($destFilePath, $uploadId, $parts) } return $result['Location']; } - - protected function doListFiles($filePath, $pathPrefix = '') + + protected function doListFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false) { $dirList = array(); list($bucket, $filePath) = $this->getBucketAndFilePath($filePath); - + try { $dirListObjectsRaw = $this->s3Client->getIterator('ListObjects', array( 'Bucket' => $bucket, 'Prefix' => $filePath )); - + $originalFilePath = $bucket . '/' . $filePath . '/'; foreach ($dirListObjectsRaw as $dirListObject) { @@ -598,21 +598,24 @@ protected function doListFiles($filePath, $pathPrefix = '') $fileName = $pathPrefix.basename($fullPath); if($originalFilePath == $fullPath) continue; - + $fileType = "file"; if($dirListObject['Size'] == 0 && substr_compare($fullPath, '/', -strlen('/')) === 0) { $fileType = 'dir'; } - + if ($fileType == 'dir') { - $dirList[] = array($fileName, 'dir', $dirListObject['Size']); - $dirList = array_merge($dirList, self::doListFiles($fullPath, $pathPrefix)); + $dirList[] = $fileNamesOnly ? $fileName : array($fileName, 'dir', $dirListObject['Size']); + if( $recursive) + { + $dirList = array_merge($dirList, self::doListFiles($fullPath, $pathPrefix, $fileNamesOnly)); + } } else { - $dirList[] = array($fileName, 'file', $dirListObject['Size']); + $dirList[] = $fileNamesOnly ? $fileName : array($fileName, 'file', $dirListObject['Size']); } } } @@ -620,10 +623,10 @@ protected function doListFiles($filePath, $pathPrefix = '') { self::safeLog("Couldn't list file objects for remote path, [$filePath] from bucket [$bucket]: {$e->getMessage()}"); } - + return $dirList; } - + protected function doGetMaximumPartsNum() { return self::MAX_PARTS_NUMBER; diff --git a/infra/storage/shared_file_system_managers/kS3SharedFileSystemMgr_V3_SDK.php b/infra/storage/shared_file_system_managers/kS3SharedFileSystemMgr_V3_SDK.php index 62bc38b0097..079b0f7f061 100644 --- a/infra/storage/shared_file_system_managers/kS3SharedFileSystemMgr_V3_SDK.php +++ b/infra/storage/shared_file_system_managers/kS3SharedFileSystemMgr_V3_SDK.php @@ -538,18 +538,18 @@ public function completeMultiPartUpload($destFilePath, $uploadId, $parts) } return $result['Location']; } - - protected function doListFiles($filePath, $pathPrefix = '') + + protected function doListFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false) { $results = $this->s3Call('listObjects', null, $filePath); - + if(!$results) { return false; } return $results; } - + protected function doGetMaximumPartsNum() { return self::MAX_PARTS_NUMBER; diff --git a/infra/storage/shared_file_system_managers/kSharedFileSystemMgr.php b/infra/storage/shared_file_system_managers/kSharedFileSystemMgr.php index 8b9714d7002..9a6ee9b09ec 100644 --- a/infra/storage/shared_file_system_managers/kSharedFileSystemMgr.php +++ b/infra/storage/shared_file_system_managers/kSharedFileSystemMgr.php @@ -251,15 +251,17 @@ abstract protected function doGetUploadMinimumSize(); * @return int */ abstract protected function doGetUploadMaxSize(); - + /** * returns list of files under given file path * * @param $filePath file path to list dir content for - * + * @param string $pathPrefix + * @param bool $recursive + * @param bool $fileNamesOnly * @return array */ - abstract protected function doListFiles($filePath, $pathPrefix = ''); + abstract protected function doListFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false); /** * returns true/false if the givven file path exists and is a regular file @@ -517,10 +519,10 @@ function copySingleFile($from, $to, $deleteSrc) return $this->doRename($from, $to); } - public function listFiles($filePath, $pathPrefix = '') + public function listFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false) { $filePath = kFileBase::fixPath($filePath); - return $this->doListFiles($filePath, $pathPrefix); + return $this->doListFiles($filePath, $pathPrefix, $recursive, $fileNamesOnly); } public function isFile($filePath)