Merge pull request #10462 from kaltura/Propus-16.17.0-FOUN-111

FOUN-111

batch/batches/Storage/Engine/KFileTransferExportEngine.php

@@ -27,23 +27,24 @@ function __construct($data, $jobSubType) {
 	function export() 
 	{
 		$srcTempFile = null;
-		
-		$srcFile = kFile::realPath($this->srcFile);
-		$realPathRemote = strpos($srcFile, "http") !== false && kFile::checkFileExists($this->srcFile);
-		if($realPathRemote || !KBatchBase::pollingFileExists($this->srcFile))
+
+		list($isRemote, $remoteUrl, $isDir) = kFile::resolveFilePath($this->srcFile);
+		$realPathRemote = $isRemote && kFile::checkFileExists($this->srcFile);
+		if( $realPathRemote || !KBatchBase::pollingFileExists($this->srcFile))
 		{
-			$externalUrl = $realPathRemote ? $srcFile : $this->data->externalUrl;
+			$externalUrl = $isRemote ? $remoteUrl : $this->data->externalUrl;
 			if($externalUrl == null)
 			{
 				throw new kTemporaryException("Source file {$this->srcFile} does not exist, and no external URL provided");
 			}
-
-			$srcTempFile = $this->getAssetFile($externalUrl, $this->data->flavorAssetId);
+
+			$srcTempFile = kFile::fetchRemoteToLocal($this->srcFile, $externalUrl, $isDir,
+				sys_get_temp_dir() . "/imports/", $this->data->flavorAssetId . "_" . basename($this->srcFile));
+
 			if(!$srcTempFile)
 			{
 				throw new kTemporaryException("Source file {$this->srcFile} does not exist");
 			}
-
 			$this->srcFile = $srcTempFile;
 		}
 
@@ -93,6 +94,7 @@ function export()
 			else if (kFile::isDir($this->srcFile))
 			{
 				$filesPaths = kFile::dirList($this->srcFile);
+
 				$destDir = $this->destFile;
 				foreach ($filesPaths as $filePath)
 				{
@@ -229,56 +231,29 @@ protected function addS3FieldsToStorageData($storageExportData, $externalStorage
 		return $storageExportData;
 	}
 
-	protected function getAssetFile($externalUrl, $uniqueDownloadId = null)
+	protected function unlinkFileIfNeeded($tempFile)
 	{
-		// Needed arguments
-		if($externalUrl === null)
-		{
-			KalturaLog::info("No external url provided,");
-			return null;
-		}
-
-		// Create the temporary file path
-		$tempDirectoryPath = sys_get_temp_dir();
-		if (!kFile::isDir($tempDirectoryPath))
-		{
-			kFile::fullMkfileDir($tempDirectoryPath, 0700, true);
-		}
-
-		if(!$uniqueDownloadId)
+		if (!$tempFile)
 		{
-			$uniqueDownloadId = uniqid(rand(),true);
+			return;
 		}
-
-		$filePath = $tempDirectoryPath . "/asset_$uniqueDownloadId";
 
-		// Retrieve the file
-		$res = null;
-		try
+		if (kFile::isDir($tempFile))
 		{
-			$stat = KCurlWrapper::getDataFromFile($externalUrl, $filePath, null, true);
-
-			if ($stat)
+			$dir = dir($tempFile);
+			if (!$dir)
 			{
-				$res = $filePath;
-				KalturaLog::info("Succeeded to retrieve asset content for assetId: [$assetId]");
+				return null;
 			}
-			else
+			while ($dirFile = $dir->read())
 			{
-				KalturaLog::info("Failed to retrieve asset content for assetId: [$assetId]");
+				if ($dirFile != "." && $dirFile != "..")
+				{
+					unlink($tempFile . DIRECTORY_SEPARATOR . $dirFile);
+				}
 			}
 		}
-		catch(Exception $e)
-		{
-			KalturaLog::info("Can't serve asset id [$assetId] from [$externalUrl] " . $e->getMessage());
-		}
-
-		return $res;
-	}
-
-	protected function unlinkFileIfNeeded($tempFile)
-	{
-		if($tempFile)
+		else
 		{
 			unlink($tempFile);
 		}

infra/storage/kFile.class.php

@@ -61,8 +61,15 @@ public static function listDir($path, $pathPrefix = '')
 	}
 
 	// TODO - implement recursion
-	static public function dirList($directory, $return_directory_as_prefix = true, $should_recurse = false)
+	static public function dirList($directory, $return_directory_as_prefix = true)
 	{
+		if (kFile::isSharedPath($directory))
+		{
+			$sharedFsMgr = kSharedFileSystemMgr::getInstanceFromPath($directory);
+			$pathPrefix = $return_directory_as_prefix ? $directory . "/" : '';
+			return $sharedFsMgr->listFiles($directory . "/" , $pathPrefix, false, true);
+		}
+
 		// create an array to hold directory list
 		$results = array();
 
@@ -74,10 +81,9 @@ static public function dirList($directory, $return_directory_as_prefix = true, $
 		// keep going until all files in directory have been read
 		while($file = readdir($handler))
 		{
-
 			// if $file isn't this directory or its parent,
 			// add it to the results array
-			if($file != '.' && $file != '..')
+			if($file != '.' && $file != '..' )
 			{
 				$results[] = ($return_directory_as_prefix ? $directory . "/" : "") . $file;
 			}

infra/storage/shared_file_system_managers/kNfsSharedFileSystemMgr.php

@@ -221,7 +221,7 @@ protected function doGetUploadMaxSize()
 		return 2000000000;
 	}
 
-	protected function doListFiles($filePath, $pathPrefix = '')
+	protected function doListFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false)
 	{
 		$fileList = array();
 		$path = str_ireplace(DIRECTORY_SEPARATOR, '/', $filePath);
@@ -238,12 +238,15 @@ protected function doListFiles($filePath, $pathPrefix = '')
 					if (is_dir($fullPath))
 					{
 						$tmpPrefix = $tmpPrefix.'/';
-						$fileList[] = array($tmpPrefix, 'dir', self::fileSize($fullPath));
-						$fileList = array_merge($fileList, self::listDir($fullPath, $tmpPrefix));
+						$fileList[] = $fileNamesOnly ? $tmpPrefix : array($tmpPrefix, 'dir', self::fileSize($fullPath));
+						if ($recursive)
+						{
+							$fileList = array_merge($fileList, self::doListFiles($fullPath, $tmpPrefix));
+						}
 					}
 					else
 					{
-						$fileList[] = array($tmpPrefix, 'file', self::fileSize($fullPath));
+						$fileList[] = $fileNamesOnly ? $tmpPrefix : array($tmpPrefix, 'file', self::fileSize($fullPath));
 					}
 				}
 			}

infra/storage/shared_file_system_managers/kS3SharedFileSystemMgr.php

@@ -578,52 +578,55 @@ public function completeMultiPartUpload($destFilePath, $uploadId, $parts)
 		}
 		return $result['Location'];
 	}
-	
-	protected function doListFiles($filePath, $pathPrefix = '')
+
+	protected function doListFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false)
 	{
 		$dirList = array();
 		list($bucket, $filePath) = $this->getBucketAndFilePath($filePath);
-		
+
 		try
 		{
 			$dirListObjectsRaw = $this->s3Client->getIterator('ListObjects', array(
 				'Bucket' => $bucket,
 				'Prefix' => $filePath
 			));
-			
+
 			$originalFilePath = $bucket . '/' . $filePath . '/';
 			foreach ($dirListObjectsRaw as $dirListObject)
 			{
 				$fullPath = '/' . $bucket . '/' . $dirListObject['Key'];
 				$fileName = $pathPrefix.basename($fullPath);
 				if($originalFilePath == $fullPath)
 					continue;
-				
+
 				$fileType = "file";
 				if($dirListObject['Size'] == 0 && substr_compare($fullPath, '/', -strlen('/')) === 0)
 				{
 					$fileType = 'dir';
 				}
-				
+
 				if ($fileType == 'dir')
 				{
-					$dirList[] = array($fileName, 'dir', $dirListObject['Size']);
-					$dirList = array_merge($dirList, self::doListFiles($fullPath, $pathPrefix));
+					$dirList[] = $fileNamesOnly ?  $fileName : array($fileName, 'dir', $dirListObject['Size']);
+					if( $recursive)
+					{
+						$dirList = array_merge($dirList, self::doListFiles($fullPath, $pathPrefix, $fileNamesOnly));
+					}
 				}
 				else
 				{
-					$dirList[] = array($fileName, 'file', $dirListObject['Size']);
+					$dirList[] = $fileNamesOnly ? $fileName : array($fileName, 'file', $dirListObject['Size']);
 				}
 			}
 		}
 		catch ( Exception $e )
 		{
 			self::safeLog("Couldn't list file objects for remote path, [$filePath] from bucket [$bucket]: {$e->getMessage()}");
 		}
-		
+
 		return $dirList;
 	}
-	
+
 	protected function doGetMaximumPartsNum()
 	{
 		return self::MAX_PARTS_NUMBER;

infra/storage/shared_file_system_managers/kS3SharedFileSystemMgr_V3_SDK.php

@@ -538,18 +538,18 @@ public function completeMultiPartUpload($destFilePath, $uploadId, $parts)
 		}
 		return $result['Location'];
 	}
-	
-	protected function doListFiles($filePath, $pathPrefix = '')
+
+	protected function doListFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false)
 	{
 		$results = $this->s3Call('listObjects', null, $filePath);
-		
+
 		if(!$results)
 		{
 			return false;
 		}
 		return $results;
 	}
-	
+
 	protected function doGetMaximumPartsNum()
 	{
 		return self::MAX_PARTS_NUMBER;

infra/storage/shared_file_system_managers/kSharedFileSystemMgr.php

@@ -251,15 +251,17 @@ abstract protected function doGetUploadMinimumSize();
 	 * @return int
 	 */
 	abstract protected function doGetUploadMaxSize();
-	
+
 	/**
 	 * returns list of files under given file path
 	 *
 	 * @param $filePath file path to list dir content for
-	 *
+	 * @param string $pathPrefix
+	 * @param bool $recursive
+	 * @param bool $fileNamesOnly
 	 * @return array
 	 */
-	abstract protected function doListFiles($filePath, $pathPrefix = '');
+	abstract protected function doListFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false);
 
 	/**
 	 * returns true/false if the givven file path exists and is a regular file
@@ -517,10 +519,10 @@ function copySingleFile($from, $to, $deleteSrc)
 		return $this->doRename($from, $to);
 	}
 
-	public function listFiles($filePath, $pathPrefix = '')
+	public function listFiles($filePath, $pathPrefix = '', $recursive = true, $fileNamesOnly = false)
 	{
 		$filePath = kFileBase::fixPath($filePath);
-		return $this->doListFiles($filePath, $pathPrefix);
+		return $this->doListFiles($filePath, $pathPrefix, $recursive, $fileNamesOnly);
 	}
 
 	public function isFile($filePath)