diff --git a/src/lib/php/Dao/LicenseDao.php b/src/lib/php/Dao/LicenseDao.php index 4f4e9bec..8b385f4a 100644 --- a/src/lib/php/Dao/LicenseDao.php +++ b/src/lib/php/Dao/LicenseDao.php @@ -1,652 +1,652 @@ dbManager = $dbManager; $this->logger = new Logger(self::className()); } /** * \brief get all the licenses for a single file or uploadtree * * @param ItemTreeBounds $itemTreeBounds * @param int * @return LicenseMatch[] */ function getAgentFileLicenseMatches(ItemTreeBounds $itemTreeBounds, $usageId=LicenseMap::TRIVIAL) { $uploadTreeTableName = $itemTreeBounds->getUploadTreeTableName(); $statementName = __METHOD__ . ".$uploadTreeTableName.$usageId"; $params = array($itemTreeBounds->getUploadId(), $itemTreeBounds->getLeft(), $itemTreeBounds->getRight()); if($usageId==LicenseMap::TRIVIAL) { $licenseJoin = "ONLY license_ref mlr ON license_file.rf_fk = mlr.rf_pk"; } else { $params[] = $usageId; $licenseMapCte = LicenseMap::getMappedLicenseRefView('$4'); $licenseJoin = "($licenseMapCte) AS mlr ON license_file.rf_fk = mlr.rf_origin"; } $this->dbManager->prepare($statementName, "SELECT LFR.rf_shortname AS license_shortname, LFR.rf_fullname AS license_fullname, LFR.rf_pk AS license_id, LFR.fl_pk AS license_file_id, LFR.pfile_fk as file_id, LFR.rf_match_pct AS percent_match, AG.agent_name AS agent_name, AG.agent_pk AS agent_id, AG.agent_rev AS agent_revision FROM ( SELECT mlr.rf_fullname, mlr.rf_shortname, mlr.rf_pk, license_file.fl_pk, license_file.agent_fk, license_file.pfile_fk, license_file.rf_match_pct FROM license_file JOIN $licenseJoin) as LFR INNER JOIN $uploadTreeTableName as UT ON UT.pfile_fk = LFR.pfile_fk INNER JOIN agent as AG ON AG.agent_pk = LFR.agent_fk WHERE AG.agent_enabled='true' and UT.upload_fk=$1 AND UT.lft BETWEEN $2 and $3 ORDER BY license_shortname ASC, percent_match DESC"); $result = $this->dbManager->execute($statementName, $params); $matches = array(); while ($row = $this->dbManager->fetchArray($result)) { $licenseRef = new LicenseRef(intval($row['license_id']), $row['license_shortname'], $row['license_fullname']); $agentRef = new AgentRef(intval($row['agent_id']), $row['agent_name'], $row['agent_revision']); $matches[] = new LicenseMatch(intval($row['file_id']), $licenseRef, $agentRef, intval($row['license_file_id']), intval($row['percent_match'])); } $this->dbManager->freeResult($result); return $matches; } /** * \brief get all the tried bulk recognitions for a single file or uploadtree (currently unused) * * @param ItemTreeBounds $itemTreeBounds * @return LicenseMatch[] */ function getBulkFileLicenseMatches(ItemTreeBounds $itemTreeBounds) { $uploadTreeTableName = $itemTreeBounds->getUploadTreeTableName(); $statementName = __METHOD__ . ".$uploadTreeTableName"; $this->dbManager->prepare($statementName, "SELECT LF.rf_shortname AS license_shortname, LF.rf_fullname AS license_fullname, LF.rf_pk AS license_id, LFB.lrb_pk AS license_file_id, LFB.removing AS removing, UT.pfile_fk as file_id FROM license_ref_bulk as LFB INNER JOIN license_ref as LF on LF.rf_pk = LFB.rf_fk INNER JOIN $uploadTreeTableName as UT ON UT.uploadtree_pk = LFB.uploadtree_fk WHERE UT.upload_fk=$1 AND UT.lft BETWEEN $2 and $3 ORDER BY license_file_id ASC"); $result = $this->dbManager->execute($statementName, array($itemTreeBounds->getUploadId(), $itemTreeBounds->getLeft(), $itemTreeBounds->getRight())); $matches = array(); while ($row = $this->dbManager->fetchArray($result)) { $licenseRef = new LicenseRef($row['license_id'], $row['license_shortname'], $row['license_fullname']); if ($row['removing'] == 'f') { $agentID = 1; $agentName = "bulk addition"; } else { $agentID = 2; $agentName = "bulk removal"; } $agentRef = new AgentRef($agentID, $agentName, "empty"); $matches[] = new LicenseMatch(intval($row['file_id']), $licenseRef, $agentRef, intval($row['license_file_id'])); } $this->dbManager->freeResult($result); return $matches; } /** * @return LicenseRef[] */ public function getLicenseRefs($search = null, $orderAscending = true) { if (isset($_SESSION) && array_key_exists('GroupId', $_SESSION)) { $rfTable = 'license_all'; $options = array('columns' => array('rf_pk', 'rf_shortname', 'rf_fullname'), 'candidatePrefix' => $this->candidatePrefix); $licenseViewDao = new LicenseViewProxy($_SESSION['GroupId'], $options, $rfTable); $withCte = $licenseViewDao->asCTE(); } else { $withCte = ''; $rfTable = 'ONLY license_ref'; } $searchCondition = $search ? "WHERE rf_shortname ilike $1" : ""; $order = $orderAscending ? "ASC" : "DESC"; $statementName = __METHOD__ . ($search ? ".search_" . $search : "") . ".order_$order"; $this->dbManager->prepare($statementName, $sql = $withCte . " select rf_pk,rf_shortname,rf_fullname from $rfTable $searchCondition order by LOWER(rf_shortname) $order"); $result = $this->dbManager->execute($statementName, $search ? array('%' . strtolower($search) . '%') : array()); $licenseRefs = array(); while ($row = $this->dbManager->fetchArray($result)) { $licenseRefs[] = new LicenseRef(intval($row['rf_pk']), $row['rf_shortname'], $row['rf_fullname']); } $this->dbManager->freeResult($result); return $licenseRefs; } /** * @return LicenseRef[] */ public function getConclusionLicenseRefs($groupId, $search = null, $orderAscending = true, $exclude=array()) { $rfTable = 'license_all'; $options = array('columns' => array('rf_pk', 'rf_shortname', 'rf_fullname'), 'candidatePrefix' => $this->candidatePrefix); $licenseViewDao = new LicenseViewProxy($groupId, $options, $rfTable); $order = $orderAscending ? "ASC" : "DESC"; $statementName = __METHOD__ . ".order_$order"; $param = array(); /* exclude license with parent, excluded child or selfexcluded */ $sql = $licenseViewDao->asCTE()." SELECT rf_pk,rf_shortname,rf_fullname FROM $rfTable WHERE NOT EXISTS (select * from license_map WHERE rf_pk=rf_fk AND rf_fk!=rf_parent)"; if($search) { $param[] = '%' . $search . '%'; $statementName .= '.search'; $sql .= " AND rf_shortname ilike $1"; } if(count($exclude)>0) { // $param[] = $exclude; $tuple = implode(',', $exclude); $statementName .= '.exclude'.$tuple; $sql .= " AND NOT EXISTS (select * from license_map WHERE rf_pk=rf_parent AND rf_fk IN ($tuple)) AND rf_pk NOT IN($tuple)"; } $this->dbManager->prepare($statementName, "$sql ORDER BY LOWER(rf_shortname) $order"); $result = $this->dbManager->execute($statementName, $param); $licenseRefs = array(); while ($row = $this->dbManager->fetchArray($result)) { $licenseRefs[] = new LicenseRef(intval($row['rf_pk']), $row['rf_shortname'], $row['rf_fullname']); } $this->dbManager->freeResult($result); return $licenseRefs; } /** * @return array */ public function getLicenseArray($groupId = null) { $statementName = __METHOD__; $rfTable = 'license_all'; $options = array('columns' => array('rf_pk', 'rf_shortname', 'rf_fullname'), 'candidatePrefix' => $this->candidatePrefix); if ($groupId === null) { $groupId = (isset($_SESSION) && array_key_exists('GroupId', $_SESSION)) ? $_SESSION['GroupId'] : 0; } $licenseViewDao = new LicenseViewProxy($groupId, $options, $rfTable); $withCte = $licenseViewDao->asCTE(); $this->dbManager->prepare($statementName, $withCte . " select rf_pk id,rf_shortname shortname,rf_fullname fullname from $rfTable order by LOWER(rf_shortname)"); $result = $this->dbManager->execute($statementName); $licenseRefs = $this->dbManager->fetchAll($result); $this->dbManager->freeResult($result); return $licenseRefs; } /** * @param ItemTreeBounds $itemTreeBounds * @param int $selectedAgentId * @param array $mask * @return array */ public function getLicenseIdPerPfileForAgentId(ItemTreeBounds $itemTreeBounds, $selectedAgentId, $includeSubfolders=true, $nameRange=array()) { $uploadTreeTableName = $itemTreeBounds->getUploadTreeTableName(); $statementName = __METHOD__ . '.' . $uploadTreeTableName; $param = array($selectedAgentId); if ($includeSubfolders) { $param[] = $itemTreeBounds->getLeft(); $param[] = $itemTreeBounds->getRight(); $condition = "lft BETWEEN $2 AND $3"; $statementName .= ".subfolders"; if(!empty($nameRange)) { $condition .= " AND ufile_name BETWEEN $4 and $5"; $param[] = $nameRange[0]; $param[] = $nameRange[1]; $statementName .= ".nameRange"; } } else { $param[] = $itemTreeBounds->getItemId(); $condition = "realparent = $2"; } if ('uploadtree_a' == $uploadTreeTableName) { $param[] = $itemTreeBounds->getUploadId(); $condition .= " AND utree.upload_fk=$".count($param); } $sql = "SELECT utree.pfile_fk as pfile_id, license_ref.rf_pk as license_id, rf_match_pct as match_percentage, CAST($1 AS INT) AS agent_id, uploadtree_pk FROM license_file, license_ref, $uploadTreeTableName utree WHERE agent_fk = $1 AND license_file.rf_fk = license_ref.rf_pk AND license_file.pfile_fk = utree.pfile_fk AND $condition ORDER BY match_percentage ASC"; $this->dbManager->prepare($statementName, $sql); $result = $this->dbManager->execute($statementName, $param); $licensesPerFileId = array(); while ($row = $this->dbManager->fetchArray($result)) { $licensesPerFileId[$row['pfile_id']][$row['license_id']] = $row; } $this->dbManager->freeResult($result); return $licensesPerFileId; } /** * @param ItemTreeBounds $itemTreeBounds * @param Array(int) $selectedAgentIds * @param bool $includeSubFolders * @param String $excluding * @param bool $ignore ignore files without license * @return array */ public function getLicensesPerFileNameForAgentId(ItemTreeBounds $itemTreeBounds, $selectedAgentIds=null, $includeSubfolders=true, $excluding='', $ignore=false) { $uploadTreeTableName = $itemTreeBounds->getUploadTreeTableName(); $statementName = __METHOD__ . '.' . $uploadTreeTableName; $param = array(); $condition = " (ufile_mode & (1<<28)) = 0"; if ($includeSubfolders) { $param[] = $itemTreeBounds->getLeft(); $param[] = $itemTreeBounds->getRight(); $condition .= " AND lft BETWEEN $1 AND $2"; $statementName .= ".subfolders"; } else { $param[] = $itemTreeBounds->getItemId(); $condition .= " AND realparent = $1"; } if ('uploadtree_a' == $uploadTreeTableName) { $param[] = $itemTreeBounds->getUploadId(); $condition .= " AND upload_fk=$".count($param); } $agentSelect = ""; if ($selectedAgentIds !== null) { $statementName .= ".".count($selectedAgentIds)."agents"; $agentSelect = "WHERE agent_fk IS NULL"; foreach($selectedAgentIds as $selectedAgentId) { $param[] = $selectedAgentId; $agentSelect .= " OR agent_fk = $".count($param); } } $sql = " SELECT ufile_name, lft, rgt, ufile_mode, rf_shortname, agent_fk FROM (SELECT ufile_name, lft, rgt, ufile_mode, pfile_fk FROM $uploadTreeTableName WHERE $condition) AS subselect1 LEFT JOIN (SELECT rf_shortname,pfile_fk,agent_fk FROM license_file, license_ref WHERE rf_fk = rf_pk) AS subselect2 ON subselect1.pfile_fk = subselect2.pfile_fk $agentSelect ORDER BY lft asc "; $this->dbManager->prepare($statementName, $sql); $result = $this->dbManager->execute($statementName, $param); $licensesPerFileName = array(); $row = $this->dbManager->fetchArray($result); $pathStack = array($row['ufile_name']); $rgtStack = array($row['rgt']); $lastLft = $row['lft']; $path = implode($pathStack,'/'); $this->addToLicensesPerFileName($licensesPerFileName, $path, $row, $ignore); while ($row = $this->dbManager->fetchArray($result)) { if (!empty($excluding) && false!==strpos("/$row[ufile_name]/", $excluding)) { $lastLft = $row['rgt'] + 1; continue; } if ($row['lft'] < $lastLft) { continue; } $this->updateStackState($pathStack, $rgtStack, $lastLft, $row); $path = implode($pathStack,'/'); $this->addToLicensesPerFileName($licensesPerFileName, $path, $row, $ignore); } $this->dbManager->freeResult($result); return array_reverse($licensesPerFileName); } private function updateStackState(&$pathStack, &$rgtStack, &$lastLft, $row) { if ($row['lft'] >= $lastLft) { while(count($rgtStack) > 0 && $row['lft'] > $rgtStack[count($rgtStack)-1]) { array_pop($pathStack); array_pop($rgtStack); } if ($row['lft'] > $lastLft) { array_push($pathStack, $row['ufile_name']); array_push($rgtStack, $row['rgt']); $lastLft = $row['lft']; } } } private function addToLicensesPerFileName(&$licensesPerFileName, $path, $row, $ignore) { if (($row['ufile_mode']&(1<<29)) ==0) { if($row['rf_shortname']) { $licensesPerFileName[$path][] = $row['rf_shortname']; } } else if (!$ignore) { $licensesPerFileName[$path] = false; } } /** * @param ItemTreeBounds $itemTreeBounds * @param null|int|int[] $agentId * @return array */ public function getLicenseHistogram(ItemTreeBounds $itemTreeBounds, $agentId=null) { $uploadTreeTableName = $itemTreeBounds->getUploadTreeTableName(); $agentText = $agentId ? (is_array($agentId) ? implode(',', $agentId) : $agentId) : '-'; $statementName = __METHOD__ . '.' . $uploadTreeTableName . ".$agentText"; $param = array($itemTreeBounds->getUploadId(), $itemTreeBounds->getLeft(), $itemTreeBounds->getRight()); $sql = "SELECT rf_shortname AS license_shortname, rf_pk, count(*) AS count, count(distinct pfile_ref.pfile_fk) as unique FROM ( SELECT license_ref.rf_shortname, license_ref.rf_pk, license_file.fl_pk, license_file.agent_fk, license_file.pfile_fk FROM license_file JOIN license_ref ON license_file.rf_fk = license_ref.rf_pk) AS pfile_ref RIGHT JOIN $uploadTreeTableName UT ON pfile_ref.pfile_fk = UT.pfile_fk"; if (is_array($agentId)) { $sql .= ' AND agent_fk=ANY($4)'; $param[] = '{' . implode(',', $agentId) . '}'; } elseif (!empty($agentId)) { $sql .= ' AND agent_fk=$4'; $param[] = $agentId; } $sql .= " WHERE (rf_shortname IS NULL OR rf_shortname NOT IN ('Void')) AND upload_fk=$1 AND (UT.lft BETWEEN $2 AND $3) AND UT.ufile_mode&(3<<28)=0 GROUP BY license_shortname, rf_pk"; $this->dbManager->prepare($statementName, $sql); $result = $this->dbManager->execute($statementName, $param); $assocLicenseHist = array(); while ($row = $this->dbManager->fetchArray($result)) { $shortname = empty($row['rf_pk']) ? self::NO_LICENSE_FOUND : $row['license_shortname']; $assocLicenseHist[$shortname] = array( 'count' => intval($row['count']), 'unique' => intval($row['unique']), 'rf_pk' => intval($row['rf_pk'])); } $this->dbManager->freeResult($result); return $assocLicenseHist; } public function getLicenseShortnamesContained(ItemTreeBounds $itemTreeBounds, $latestSuccessfulAgentIds=null, $filterLicenses = array('VOID')) //'No_license_found', { $uploadTreeTableName = $itemTreeBounds->getUploadTreeTableName(); $noLicenseFoundStmt = empty($filterLicenses) ? "" : " AND rf_shortname NOT IN (" . implode(", ", array_map(function ($name) { return "'" . $name . "'"; }, $filterLicenses)) . ")"; $statementName = __METHOD__ . '.' . $uploadTreeTableName; $agentFilter = ''; if(is_array($latestSuccessfulAgentIds)) { $agentIdSet = "{" . implode(',', $latestSuccessfulAgentIds) . "}"; $statementName .= ".$agentIdSet"; $agentFilter = " AND agent_fk=ANY('$agentIdSet')"; } $this->dbManager->prepare($statementName, "SELECT license_ref.rf_shortname FROM license_file JOIN license_ref ON license_file.rf_fk = license_ref.rf_pk INNER JOIN $uploadTreeTableName uploadTree ON uploadTree.pfile_fk=license_file.pfile_fk WHERE upload_fk=$1 AND lft BETWEEN $2 AND $3 $noLicenseFoundStmt $agentFilter GROUP BY rf_shortname ORDER BY rf_shortname ASC"); $result = $this->dbManager->execute($statementName, array($itemTreeBounds->getUploadId(), $itemTreeBounds->getLeft(), $itemTreeBounds->getRight())); $licenses = array(); while ($row = $this->dbManager->fetchArray($result)) { $licenses[] = $row['rf_shortname']; } $this->dbManager->freeResult($result); return $licenses; } /** * @param string $condition * @param array $param * @param $groupId * @return License|null */ private function getLicenseByCondition($condition, $param, $groupId=null) { $row = $this->dbManager->getSingleRow( - "SELECT rf_pk, rf_shortname, rf_fullname, rf_text, rf_url, rf_risk FROM ONLY license_ref WHERE $condition", + "SELECT rf_pk, rf_shortname, rf_fullname, rf_text, rf_url, rf_risk, rf_spdx_compatible FROM ONLY license_ref WHERE $condition", $param, __METHOD__ . ".$condition.only"); if (false === $row && isset($groupId)) { $param[] = $groupId; $row = $this->dbManager->getSingleRow( - "SELECT rf_pk, rf_shortname, rf_fullname, rf_text, rf_url, rf_risk FROM license_candidate WHERE $condition AND group_fk=$".count($param), + "SELECT rf_pk, rf_shortname, rf_fullname, rf_text, rf_url, rf_risk, rf_spdx_compatible FROM license_candidate WHERE $condition AND group_fk=$".count($param), $param, __METHOD__ . ".$condition.group"); } if (false === $row) { return null; } - $license = new License(intval($row['rf_pk']), $row['rf_shortname'], $row['rf_fullname'], $row['rf_risk'], $row['rf_text'], $row['rf_url']); + $license = new License(intval($row['rf_pk']), $row['rf_shortname'], $row['rf_fullname'], $row['rf_risk'], $row['rf_text'], $row['rf_url'], $row['rf_spdx_compatible']); return $license; } /** * @param string $licenseId * @param int|null $groupId * @return License|null */ public function getLicenseById($licenseId, $groupId=null) { return $this->getLicenseByCondition('rf_pk=$1', array($licenseId), $groupId); } /** * @param string $licenseShortname * @param int|null $groupId * @return License|null */ public function getLicenseByShortName($licenseShortname, $groupId=null) { return $this->getLicenseByCondition('rf_shortname=$1', array($licenseShortname), $groupId); } /** * @param int $userId * @param int $groupId * @param int $uploadTreeId * @param bool[] $licenseRemovals * @param string $refText * @return int lrp_pk on success or -1 on fail */ public function insertBulkLicense($userId, $groupId, $uploadTreeId, $licenseRemovals, $refText) { $licenseRefBulkIdResult = $this->dbManager->getSingleRow( "INSERT INTO license_ref_bulk (user_fk, group_fk, uploadtree_fk, rf_text) VALUES ($1,$2,$3,$4) RETURNING lrb_pk", array($userId, $groupId, $uploadTreeId, $refText), __METHOD__ . '.getLrb' ); if ($licenseRefBulkIdResult === false) { return -1; } $bulkId = $licenseRefBulkIdResult['lrb_pk']; $stmt = __METHOD__ . '.insertAction'; $this->dbManager->prepare($stmt, "INSERT INTO license_set_bulk (lrb_fk, rf_fk, removing) VALUES ($1,$2,$3)"); foreach($licenseRemovals as $licenseId=>$removing) { $this->dbManager->execute($stmt, array($bulkId, $licenseId, $this->dbManager->booleanToDb($removing))); } return $bulkId ; } /** * @param string $newShortname * @param int $groupId * @return bool */ public function isNewLicense($newShortname, $groupId) { $licenceViewDao = new LicenseViewProxy($groupId, array('columns' => array('rf_shortname'))); $sql = 'SELECT count(*) cnt FROM (' . $licenceViewDao->getDbViewQuery() . ') AS license_all WHERE rf_shortname=$1'; $duplicatedRef = $this->dbManager->getSingleRow($sql, array($newShortname), __METHOD__.".$groupId" ); return $duplicatedRef['cnt'] == 0; } /** * @param string $newShortname * @param string $refText * @return int Id of license candidate */ public function insertUploadLicense($newShortname, $refText) { $sql = 'INSERT INTO license_candidate (group_fk,rf_shortname,rf_fullname,rf_text,rf_md5,rf_detector_type) VALUES ($1,$2,$2,$3,md5($3),1) RETURNING rf_pk'; $refArray = $this->dbManager->getSingleRow($sql, array($_SESSION['GroupId'], $newShortname, $refText), __METHOD__); return $refArray['rf_pk']; } public function getLicenseCount() { $licenseRefTable = $this->dbManager->getSingleRow("SELECT COUNT(*) cnt FROM license_ref WHERE rf_text!=$1", array("License by Nomos.")); return intval($licenseRefTable['cnt']); } public function updateCandidate($rf_pk, $shortname, $fullname, $rfText, $url, $readyformerge, $riskLvl) { $marydone = $this->dbManager->booleanToDb($readyformerge); $this->dbManager->getSingleRow('UPDATE license_candidate SET rf_shortname=$2, rf_fullname=$3, rf_text=$4, rf_url=$5, marydone=$6, rf_risk=$7 WHERE rf_pk=$1', array($rf_pk, $shortname, $fullname, $rfText, $url, $marydone, $riskLvl), __METHOD__); } public function getLicenseParentById($licenseId, $groupId=null) { return $this->getLicenseByCondition(" rf_pk=(SELECT rf_parent FROM license_map WHERE usage=$1 AND rf_fk=$2 AND rf_fk!=rf_parent)", array(LicenseMap::CONCLUSION,$licenseId), $groupId); } } diff --git a/src/lib/php/Data/License.php b/src/lib/php/Data/License.php index bb3ac9d6..4921214b 100644 --- a/src/lib/php/Data/License.php +++ b/src/lib/php/Data/License.php @@ -1,74 +1,87 @@ text = $text; $this->url = $url; $this->risk = $risk; + $this->spdxCompatible = $spdxCompatible; } /** * @return int */ public function getRisk() { return $this->risk; } + /** + * @return boolean + */ + public function getSpdxCompatible() + { + return $this->spdxCompatible; + } + /** * @return string */ public function getText() { return $this->text; } /** * @return string */ public function getUrl() { return $this->url; } /** @return LicenseRef */ public function getRef() { return new parent($this->getId(), $this->getShortName(), $this->getFullName()); } -} \ No newline at end of file +} diff --git a/src/spdx2/agent/spdx2.php b/src/spdx2/agent/spdx2.php index bcd33a0e..6a733af0 100644 --- a/src/spdx2/agent/spdx2.php +++ b/src/spdx2/agent/spdx2.php @@ -1,663 +1,675 @@ 'N', 'monk' => 'M'); /** @var array */ protected $includedLicenseIds = array(); /** @var string */ protected $uri; /** @var string */ protected $outputFormat = self::DEFAULT_OUTPUT_FORMAT; function __construct() { parent::__construct('spdx2', AGENT_VERSION, AGENT_REV); $this->uploadDao = $this->container->get('dao.upload'); $this->clearingDao = $this->container->get('dao.clearing'); + $this->licenseDao = $this->container->get('dao.license'); $this->dbManager = $this->container->get('db.manager'); $this->renderer = $this->container->get('twig.environment'); $this->renderer->setCache(false); $this->agentSpecifLongOptions[] = self::UPLOAD_ADDS.':'; $this->agentSpecifLongOptions[] = self::OUTPUT_FORMAT_KEY.':'; } /** * @param string[] $args * * @return string[] $args */ protected function preWorkOnArgs($args) { if ((!array_key_exists(self::OUTPUT_FORMAT_KEY,$args) || $args[self::OUTPUT_FORMAT_KEY] === "") && array_key_exists(self::UPLOAD_ADDS,$args)) { $args = SpdxTwoUtils::preWorkOnArgsFlp($args,self::UPLOAD_ADDS,self::OUTPUT_FORMAT_KEY); } else { if (!array_key_exists(self::UPLOAD_ADDS,$args) || $args[self::UPLOAD_ADDS] === "") { $args = SpdxTwoUtils::preWorkOnArgsFlp($args,self::UPLOAD_ADDS,self::OUTPUT_FORMAT_KEY); } } return $args; } /** * @param int $uploadId * @return bool */ function processUploadId($uploadId) { $args = $this->preWorkOnArgs($this->args); if(array_key_exists(self::OUTPUT_FORMAT_KEY,$args)) { $possibleOutputFormat = trim($args[self::OUTPUT_FORMAT_KEY]); if(in_array($possibleOutputFormat, explode(',',self::AVAILABLE_OUTPUT_FORMATS))) { $this->outputFormat = $possibleOutputFormat; } } $this->licenseMap = new LicenseMap($this->dbManager, $this->groupId, LicenseMap::REPORT, true); $this->computeUri($uploadId); $packageNodes = $this->renderPackage($uploadId); $additionalUploadIds = array_key_exists(self::UPLOAD_ADDS,$args) ? explode(',',$args[self::UPLOAD_ADDS]) : array(); $packageIds = array($uploadId); foreach($additionalUploadIds as $additionalId) { $packageNodes .= $this->renderPackage($additionalId); $packageIds[] = $additionalId; } $this->writeReport($packageNodes, $packageIds, $uploadId); return true; } /** * @param string $partname * @return string */ protected function getTemplateFile($partname) { $prefix = $this->outputFormat . "-"; $postfix = ".twig"; switch ($this->outputFormat) { case "spdx2": $postfix = ".xml" . $postfix; break; case "spdx2tv": break; case "dep5": $prefix = $prefix . "copyright-"; break; } return $prefix . $partname . $postfix; } /** * @param string $fileBase * @param string $packageName * @return string */ protected function getUri($fileBase,$packageName) { $fileName = $fileBase. strtoupper($this->outputFormat)."_".$packageName.'_'.time(); switch ($this->outputFormat) { case "spdx2": $fileName = $fileName .".rdf" ; break; case "spdx2tv": $fileName = $fileName .".spdx" ; break; case "dep5": $fileName = $fileName .".txt" ; break; } return $fileName; } /** * @param int $uploadId * @return string */ protected function renderPackage($uploadId) { $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId); $itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId,$uploadTreeTableName); $this->heartbeat(0); $filesWithLicenses = $this->getFilesWithLicensesFromClearings($itemTreeBounds); $this->heartbeat(0); $this->addClearingStatus($filesWithLicenses,$itemTreeBounds); $this->heartbeat(0); $licenseComment = $this->addScannerResults($filesWithLicenses, $itemTreeBounds); $this->heartbeat(0); $this->addCopyrightResults($filesWithLicenses, $uploadId); $this->heartbeat(0); $upload = $this->uploadDao->getUpload($uploadId); $fileNodes = $this->generateFileNodes($filesWithLicenses, $upload->getTreeTableName()); $mainLicenseIds = $this->clearingDao->getMainLicenseIds($uploadId, $this->groupId); $mainLicenses = array(); foreach($mainLicenseIds as $licId) { $reportedLicenseId = $this->licenseMap->getProjectedId($licId); $this->includedLicenseIds[$reportedLicenseId] = true; $mainLicenses[] = $this->licenseMap->getProjectedShortname($reportedLicenseId); } $hashes = $this->uploadDao->getUploadHashes($uploadId); return $this->renderString($this->getTemplateFile('package'),array( 'uploadId'=>$uploadId, 'uri'=>$this->uri, 'packageName'=>$upload->getFilename(), 'uploadName'=>$upload->getFilename(), 'sha1'=>$hashes['sha1'], 'md5'=>$hashes['md5'], 'verificationCode'=>$this->getVerificationCode($upload), 'mainLicenses'=>$mainLicenses, 'mainLicense'=>SpdxTwoUtils::implodeLicenses($mainLicenses, "LicenseRef-"), 'licenseComments'=>$licenseComment, 'fileNodes'=>$fileNodes) ); } /** * @param ItemTreeBounds $itemTreeBounds * @return string[][][] $filesWithLicenses mapping item->'concluded'->(array of shortnames) */ protected function getFilesWithLicensesFromClearings(ItemTreeBounds $itemTreeBounds) { $clearingDecisions = $this->clearingDao->getFileClearingsFolder($itemTreeBounds, $this->groupId); $filesWithLicenses = array(); $clearingsProceeded = 0; foreach ($clearingDecisions as $clearingDecision) { $clearingsProceeded += 1; if(($clearingsProceeded&2047)==0) { $this->heartbeat(0); } if($clearingDecision->getType() == DecisionTypes::IRRELEVANT) { continue; } foreach ($clearingDecision->getClearingEvents() as $clearingEvent) { $clearingLicense = $clearingEvent->getClearingLicense(); if ($clearingLicense->isRemoved()) { continue; } + $spdxCheck = $this->dbManager->booleanFromDb($this->licenseDao->getLicenseById($clearingLicense->getLicenseId(), $this->groupId)->getSpdxCompatible()); + if($spdxCheck){ + $prefix = ""; + }else{ + $prefix = "LicenseRef-"; + } if($clearingEvent->getReportinfo()) { $customLicenseText = $clearingEvent->getReportinfo(); $reportedLicenseShortname = $this->licenseMap->getProjectedShortname($this->licenseMap->getProjectedId($clearingLicense->getLicenseId())) . '-' . md5($customLicenseText); $this->includedLicenseIds[$reportedLicenseShortname] = $customLicenseText; $filesWithLicenses[$clearingDecision->getUploadTreeId()]['concluded'][] = $reportedLicenseShortname; + $filesWithLicenses[$clearingDecision->getUploadTreeId()]['prefix'][$reportedLicenseShortname] = $prefix; } else { $reportedLicenseId = $this->licenseMap->getProjectedId($clearingLicense->getLicenseId()); $this->includedLicenseIds[$reportedLicenseId] = true; - $filesWithLicenses[$clearingDecision->getUploadTreeId()]['concluded'][] = - $this->licenseMap->getProjectedShortname($reportedLicenseId); + $filesWithLicenses[$clearingDecision->getUploadTreeId()]['concluded'][] = $this->licenseMap->getProjectedShortname($reportedLicenseId); + $filesWithLicenses[$clearingDecision->getUploadTreeId()]['prefix'][$this->licenseMap->getProjectedShortname($reportedLicenseId)] = $prefix; } } } return $filesWithLicenses; } /** * @param string[][][] $filesWithLicenses * @param string[] $licenses * @param string[] $copyrights * @param string $file * @param string $fullPath */ protected function toLicensesWithFilesAdder(&$filesWithLicenses, $licenses, $copyrights, $file, $fullPath) { $key = SpdxTwoUtils::implodeLicenses($licenses); if (!array_key_exists($key, $filesWithLicenses)) { $filesWithLicenses[$key]['files']=array(); $filesWithLicenses[$key]['copyrights']=array(); } $filesWithLicenses[$key]['files'][$file] = $fullPath; foreach ($copyrights as $copyright) { if (!in_array($copyright, $filesWithLicenses[$key]['copyrights'])) { $filesWithLicenses[$key]['copyrights'][] = $copyright; } } } /** * @param string[][][] $filesWithLicenses * @param string $treeTableName */ protected function toLicensesWithFiles(&$filesWithLicenses, $treeTableName) { $licensesWithFiles = array(); $treeDao = $this->container->get('dao.tree'); $filesProceeded = 0; foreach($filesWithLicenses as $fileId=>$licenses) { $filesProceeded += 1; if(($filesProceeded&2047)==0) { $this->heartbeat(0); } $fullPath = $treeDao->getFullPath($fileId,$treeTableName); if(!empty($licenses['concluded']) && count($licenses['concluded'])>0) { $this->toLicensesWithFilesAdder($licensesWithFiles,$licenses['concluded'],$licenses['copyrights'],$fileId,$fullPath); } else { if(!empty($licenses['scanner']) && count($licenses['scanner'])>0) { $implodedLicenses = SpdxTwoUtils::implodeLicenses($licenses['scanner']); if($licenses['isCleared']) { $msgLicense = "None (scanners found: " . $implodedLicenses . ")"; } else { $msgLicense = "NoLicenseConcluded (scanners found: " . $implodedLicenses . ")"; } } else { if($licenses['isCleared']) { $msgLicense = "None"; } else { $msgLicense = "NoLicenseConcluded"; } } $this->toLicensesWithFilesAdder($licensesWithFiles,array($msgLicense),$licenses['copyrights'],$fileId,$fullPath); } } return $licensesWithFiles; } /** * @param string[][][] &$filesWithLicenses * @param ItemTreeBounds $itemTreeBounds */ protected function addScannerResults(&$filesWithLicenses, ItemTreeBounds $itemTreeBounds) { $uploadId = $itemTreeBounds->getUploadId(); $scannerAgents = array_keys($this->agentNames); $scanJobProxy = new ScanJobProxy($this->container->get('dao.agent'), $uploadId); $scanJobProxy->createAgentStatus($scannerAgents); $scannerIds = $scanJobProxy->getLatestSuccessfulAgentIds(); if(empty($scannerIds)) { return; } $selectedScanners = '{'.implode(',',$scannerIds).'}'; $tableName = $itemTreeBounds->getUploadTreeTableName(); $stmt = __METHOD__ .'.scanner_findings'; $sql = "SELECT DISTINCT uploadtree_pk,rf_fk FROM $tableName ut, license_file WHERE ut.pfile_fk=license_file.pfile_fk AND rf_fk IS NOT NULL AND agent_fk=any($1)"; $param = array($selectedScanners); if ($tableName == 'uploadtree_a') { $param[] = $uploadId; $sql .= " AND upload_fk=$".count($param); $stmt .= $tableName; } $sql .= " GROUP BY uploadtree_pk,rf_fk"; $this->dbManager->prepare($stmt, $sql); $res = $this->dbManager->execute($stmt,$param); while($row=$this->dbManager->fetchArray($res)) { $reportedLicenseId = $this->licenseMap->getProjectedId($row['rf_fk']); $shortName = $this->licenseMap->getProjectedShortname($reportedLicenseId); if ($shortName != 'No_license_found' && $shortName != 'Void') { $filesWithLicenses[$row['uploadtree_pk']]['scanner'][] = $shortName; $this->includedLicenseIds[$reportedLicenseId] = true; } } $this->dbManager->freeResult($res); return "licenseInfoInFile determined by Scanners $selectedScanners"; } /** * @param string[][][] &$filesWithLicenses * @param int $uploadId * @return string */ protected function addCopyrightResults(&$filesWithLicenses, $uploadId) { /* @var $copyrightDao CopyrightDao */ $copyrightDao = $this->container->get('dao.copyright'); $uploadtreeTable = $this->uploadDao->getUploadtreeTableName($uploadId); $allEntries = $copyrightDao->getAllEntries('copyright', $uploadId, $uploadtreeTable, $type='statement'); foreach ($allEntries as $finding) { $filesWithLicenses[$finding['uploadtree_pk']]['copyrights'][] = \convertToUTF8($finding['content'],false); } } /** * @param string[][][] &$filesWithLicenses * @param ItemTreeBounds $itemTreeBounds */ protected function addClearingStatus(&$filesWithLicenses,ItemTreeBounds $itemTreeBounds) { $alreadyClearedUploadTreeView = new UploadTreeProxy($itemTreeBounds->getUploadId(), array(UploadTreeProxy::OPT_SKIP_THESE => UploadTreeProxy::OPT_SKIP_ALREADY_CLEARED, UploadTreeProxy::OPT_ITEM_FILTER => "AND (lft BETWEEN ".$itemTreeBounds->getLeft()." AND ".$itemTreeBounds->getRight().")", UploadTreeProxy::OPT_GROUP_ID => $this->groupId), $itemTreeBounds->getUploadTreeTableName(), 'already_cleared_uploadtree' . $itemTreeBounds->getUploadId()); $alreadyClearedUploadTreeView->materialize(); $filesThatShouldStillBeCleared = $alreadyClearedUploadTreeView->getNonArtifactDescendants($itemTreeBounds); $alreadyClearedUploadTreeView->unmaterialize(); $uploadTreeIds = array_keys($filesWithLicenses); foreach($uploadTreeIds as $uploadTreeId) { $filesWithLicenses[$uploadTreeId]['isCleared'] = false == array_key_exists($uploadTreeId,$filesThatShouldStillBeCleared); } } /** * @param int $uploadId */ protected function computeUri($uploadId) { global $SysConf; $upload = $this->uploadDao->getUpload($uploadId); $packageName = $upload->getFilename(); $fileBase = $SysConf['FOSSOLOGY']['path']."/report/"; $this->uri = $this->getUri($fileBase,$packageName); } /** * @param string[] $packageNodes * @param int[] $packageIds * @param int $uploadId */ protected function writeReport(&$packageNodes, $packageIds, $uploadId) { $fileBase = dirname($this->uri); if(!is_dir($fileBase)) { mkdir($fileBase, 0777, true); } umask(0133); $message = $this->renderString($this->getTemplateFile('document'),array( 'documentName'=>$fileBase, 'uri'=>$this->uri, 'userName'=>$this->container->get('dao.user')->getUserName($this->userId), 'organisation'=>'', 'packageNodes'=>$packageNodes, 'packageIds'=>$packageIds, 'licenseTexts'=>$this->getLicenseTexts()) ); // To ensure the file is valid, replace any non-printable characters with a question mark. // 'Non-printable' is ASCII < 0x20 (excluding \r, \n and tab) and 0x7F (delete). $message = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/','?',$message); file_put_contents($this->uri, $message); $this->updateReportTable($uploadId, $this->jobId, $this->uri); } /** * @param int $uploadId * @param int $jobId * @param string $fileName */ protected function updateReportTable($uploadId, $jobId, $fileName){ $this->dbManager->insertTableRow('reportgen', array('upload_fk'=>$uploadId, 'job_fk'=>$jobId, 'filepath'=>$fileName), __METHOD__); } /** * @param string $templateName * @param array $vars * @return string */ protected function renderString($templateName, $vars) { return $this->renderer->loadTemplate($templateName)->render($vars); } protected function generateFileNodes($filesWithLicenses, $treeTableName) { if (strcmp($this->outputFormat, "dep5")!==0) { return $this->generateFileNodesByFiles($filesWithLicenses, $treeTableName); } else { return $this->generateFileNodesByLicenses($filesWithLicenses, $treeTableName); } } /** * @param string[][][] &$filesWithLicenses * @param string $treeTableName * @return string */ protected function generateFileNodesByFiles($filesWithLicenses, $treeTableName) { /* @var $treeDao TreeDao */ $treeDao = $this->container->get('dao.tree'); $filesProceeded = 0; $lastValue = 0; $content = ''; foreach($filesWithLicenses as $fileId=>$licenses) { $filesProceeded += 1; if(($filesProceeded&2047)==0) { $this->heartbeat($filesProceeded - $lastValue); $lastValue = $filesProceeded; } $hashes = $treeDao->getItemHashes($fileId); $fileName = $treeDao->getFullPath($fileId,$treeTableName); $content .= $this->renderString($this->getTemplateFile('file'),array( 'fileId'=>$fileId, 'sha1'=>$hashes['sha1'], 'md5'=>$hashes['md5'], 'uri'=>$this->uri, 'fileName'=>$fileName, 'fileDirName'=>dirname($fileName), 'fileBaseName'=>basename($fileName), 'isCleared'=>$licenses['isCleared'], - 'concludedLicense'=>SpdxTwoUtils::implodeLicenses($licenses['concluded'], "LicenseRef-"), + 'concludedLicense'=>SpdxTwoUtils::implodeLicenses($licenses['concluded'], $licenses['prefix']), 'concludedLicenses'=>$licenses['concluded'], 'scannerLicenses'=>$licenses['scanner'], 'copyrights'=>$licenses['copyrights'])); } $this->heartbeat($filesProceeded - $lastValue); return $content; } /** * @param string[][][] &$filesWithLicenses * @param string $treeTableName * @return string */ protected function generateFileNodesByLicenses($filesWithLicenses, $treeTableName) { $licensesWithFiles = $this->toLicensesWithFiles($filesWithLicenses, $treeTableName); $content = ''; $filesProceeded = 0; $lastStep = 0; $lastValue = 0; foreach($licensesWithFiles as $licenseId=>$entry) { $filesProceeded += count($entry['files']); if($filesProceeded&(~2047) > $lastStep) { $this->heartbeat($filesProceeded - $lastValue); $lastStep = $filesProceeded&(~2047) + 2048; $lastValue = $filesProceeded; } $comment = ""; if (strrpos($licenseId, "NoLicenseConcluded (scanners found: ", -strlen($licenseId)) !== false) { $comment = substr($licenseId,20,strlen($licenseId)-21); $licenseId = "NoLicenseConcluded"; } elseif (strrpos($licenseId, "None (scanners found: ", -strlen($licenseId)) !== false) { $comment = substr($licenseId,6,strlen($licenseId)-7); $licenseId = "None"; } $content .= $this->renderString($this->getTemplateFile('file'),array( 'fileNames'=>$entry['files'], 'license'=>$licenseId, 'copyrights'=>$entry['copyrights'], 'comment'=>$comment)); } $this->heartbeat($filesProceeded - $lastValue); return $content; } /** * @return string[] with keys being shortname */ protected function getLicenseTexts() { $licenseTexts = array(); $licenseViewProxy = new LicenseViewProxy($this->groupId,array(LicenseViewProxy::OPT_COLUMNS=>array('rf_pk','rf_shortname','rf_text'))); $this->dbManager->prepare($stmt=__METHOD__, $licenseViewProxy->getDbViewQuery()); $res = $this->dbManager->execute($stmt); while($row=$this->dbManager->fetchArray($res)) { if (array_key_exists($row['rf_pk'], $this->includedLicenseIds)) { $licenseTexts[$row['rf_shortname']] = $row['rf_text']; } } foreach($this->includedLicenseIds as $license => $customText) { if (true !== $customText) { $licenseTexts[$license] = $customText; } } $this->dbManager->freeResult($res); return $licenseTexts; } /** * @param UploadTree $upload * @return string */ protected function getVerificationCode(Upload $upload) { $stmt = __METHOD__; $param = array(); if ($upload->getTreeTableName()=='uploadtree_a') { $sql = $upload->getTreeTableName().' WHERE upload_fk=$1 AND'; $param[] = $upload->getId(); } else { $sql = $upload->getTreeTableName().' WHERE'; $stmt .= '.'.$upload->getTreeTableName(); } $sql = "SELECT STRING_AGG(lower_sha1,'') concat_sha1 FROM (SELECT LOWER(pfile_sha1) lower_sha1 FROM pfile, $sql pfile_fk=pfile_pk ORDER BY pfile_sha1) templist"; $filelistPack = $this->dbManager->getSingleRow($sql,$param,$stmt); return sha1($filelistPack['concat_sha1']); } } $agent = new SpdxTwoAgent(); $agent->scheduler_connect(); $agent->run_scheduler_event_loop(); $agent->scheduler_disconnect(0); diff --git a/src/spdx2/agent/spdx2utils.php b/src/spdx2/agent/spdx2utils.php index f97a6935..d7b5f2fa 100644 --- a/src/spdx2/agent/spdx2utils.php +++ b/src/spdx2/agent/spdx2utils.php @@ -1,86 +1,89 @@