com web/rmtools: properly fetch providers and packages json, save and compare locally: client/include/PickleDb.php client/include/PickleWeb.php
client/script/pickle_ctl.php

Commit: cb5a1222926758f5c2a70bf280bd77934116a4d3 Author: Anatol Belski <ab@php.net> Tue, 12 May 2015 13:42:24 +0200 Parents: 4a8b33dbc3b054bc8572ad71b7230fbc71af39c6 Branches: master pecl_legacy Link: http://git.php.net/?p=web/rmtools.git;a=commitdiff;h=cb5a1222926758f5c2a70bf280bd77934116a4d3 Log: properly fetch providers and packages json, save and compare locally extension updates to go Changed paths: M client/include/PickleDb.php M client/include/PickleWeb.php M client/script/pickle_ctl.php

diff --git a/client/include/PickleDb.php b/client/include/PickleDb.php index ea0c1f2..621e387 100644 --- a/client/include/PickleDb.php +++ b/client/include/PickleDb.php @@ -4,112 +4,84 @@ namespace rmtools; class PickleDb extends \SQLite3 { - public function __construct($db_path, $autoinit = true) + protected $db_path; + + public function __construct($db_path) { - $flags = SQLITE3_OPEN_READWRITE; - $existent = file_exists($db_path); + $this->db_path = $this->createDir($db_path); + } + + protected function isSlash($c) + { + return '\\' == $c || '/' == $c; + } - if (!$existent) { - $flags |= SQLITE3_OPEN_CREATE; - } - - $this->open($db_path, $flags); - if (!$existent && $autoinit) { - $this->initDb(); + protected function buildUriLocalPath($uri) + { + if (!$this->isSlash($uri[0])) { + $uri = DIRECTORY_SEPARATOR . $uri; } + $uri = str_replace('/', DIRECTORY_SEPARATOR, $uri); + + return $this->db_path . $uri; } - public function initDb() + public function uriExists($uri) { - /* no primary keys here, reling on the hashes delivered by pickleweb. */ - $sql = "CREATE TABLE package_release (package_hash STRING, package_name STRING, ts_placed INTEGER, ts_finished INTEGER);"; - $this->exec($sql); + return file_exists($this->buildUriLocalPath($uri)); } - public function add($name, $hash, $force = false) + public function getUri($uri) { - if ($force) { - $this->remove($name, $hash); - } + $fname = $this->buildUriLocalPath($uri); - if ($this->exists($name, $hash)) { + if (!file_exists($fname)) { return false; } - $name = $this->escapeString($name); - $hash = $this->escapeString($hash); - $sql = "INSERT INTO package_release (package_name, package_hash, ts_placed, ts_finished) VALUES ('$name', '$hash', " . time() . ", 0);"; - $this->exec($sql); - - return true; + return file_get_contents($fname); } - public function remove($name, $hash) + public function saveUriJson($uri, $data) { - $name = $this->escapeString($name); - $hash = $this->escapeString($hash); - $sql = "DELETE FROM package_release WHERE package_name = '$name' AND package_hash = '$hash';"; - $this->exec($sql); + $json = json_encode($data); + + return $this->saveUri($uri, $json); } - public function exists($name, $hash, $where = '') + public function saveUri($uri, $data) { - /* cant check such thing, so trust :) */ - if ($where) { - $where = "AND $where"; - } + $fname = $this->buildUriLocalPath($uri); - $name = $this->escapeString($name); - $hash = $this->escapeString($hash); - $sql = "SELECT ts_finished FROM package_release WHERE package_name = '$name' AND package_hash = '$hash' $where;"; - - $res = $this->query($sql); - - $ret = false !== $res->fetchArray(SQLITE3_NUM); - //return $res->numColumns() > 0; - - $res->finalize(); + $dir = dirname($fname); + if (!is_dir($dir)) { + $this->createDir($dir); + } - return $ret; + return strlen($data) == file_put_contents($fname, $data); } - public function done($name, $hash) + public function getUriJson($uri) { - /* XXX That's an assumption as the latest timestamp should be about 30 mit old. - Need to extend pecl.php to set the real statuses when in't really done */ - return $this->exists($name, $hash, "ts_finished - " . time() . " > 1800"); + return json_decode($this->getUri($uri), true); } - public function dump($where = '') + public function delUri($uri) { - /* cant check such thing, so trust :) */ - if ($where) { - $where = "WHERE $where"; - } + $fname = $this->buildUriLocalPath($uri); - $res = $this->query("SELECT * FROM package_release $where ORDER BY package_name, package_hash ASC"); - echo "DUMP package_release " . PHP_EOL . PHP_EOL; - while(false !== ($row = $res->fetchArray(SQLITE3_ASSOC))) { - foreach ($row as $col => $val) { - echo "$col=$val" . PHP_EOL; - } - echo PHP_EOL; - } - $res->finalize(); + return unlink($fname); } - public function dumpQueue() + protected function createDir($path, $rec = true) { - $this->dump("ts_finished <= 0"); - } + if (!is_dir($path)) { + if (!mkdir($path, 0777, $rec)) { + throw new \Exception("failed to create '$path'"); + } + } - public function finished($name, $hash) - { - $name = $this->escapeString($name); - $hash = $this->escapeString($hash); - $sql = "UPDATE package_release SET ts_finished=" . time() . " WHERE lower(package_name) = lower('$name') AND lower(package_hash) = lower('$hash');"; - $this->exec($sql); + return realpath($path); } } - diff --git a/client/include/PickleWeb.php b/client/include/PickleWeb.php index 88c474d..8a3c5f9 100644 --- a/client/include/PickleWeb.php +++ b/client/include/PickleWeb.php @@ -4,60 +4,75 @@ namespace rmtools; class PickleWeb { - protected $db_path; + protected $db; protected $host; protected $info; protected $updatesAvailableFlag = false; - public function __construct($host, $db_path) + public function __construct($host, PickleDB $db) { $this->host = $host; - - if (!is_dir($db_path)) { - if (!mkdir($db_path)) { - throw new \Exception("failed to create '$db_path'"); - } - } - - $this->db_path = $db_path; + $this->db = $db; $this->init(); } - public function init() + protected function fetchUri($uri, $allow_empty = false) { - $uri = "{$this->host}/packages.json"; - $__tmp = file_get_contents($uri); - if (!$__tmp) { + + $url = "{$this->host}$uri"; + + $__tmp = file_get_contents($url); + + + if (false === $__tmp) { + throw new \Exception("Error encountered while receiving '$url'"); + } + + if (!$allow_empty && !$__tmp) { throw new \Exception("Empty content received from '$url'"); } - $this->info = json_decode($__tmp, true); - if (!$this->info) { - throw new \Exception("Couldn't decode JSON from '$url'"); + return $__tmp; + } + + protected function fetchUriJson($uri, $allow_empty = false) + { + $__tmp = $this->fetchUri($uri, $allow_empty); + + $ret = json_decode($__tmp, true); + if (!$ret) { + throw new \Exception("Couldn't decode JSON from '$uri' on '{$this->host}'"); } + return $ret; + } + + public function init() + { + $uri = "/packages.json"; + $this->info = $this->fetchUriJson($uri); + if (!isset($this->info["provider-includes"])) { throw new \Exception("No provider includes found"); } - $packages_json = $this->db_path . DIRECTORY_SEPARATOR . "packages.json"; - if (!file_exists($packages_json)) { + if (!$this->db->uriExists($uri)) { $this->updatesAvailableFlag = true; } else { - $packages_info = json_decode(file_get_contents($packages_json), true); - foreach ($this->info["provider-includes"] as $uri => $hash) { - if (!isset($packages_info["provider-includes"][$uri]) || - $packages_info["provider-includes"][$uri] != $hash) { + $packages_info = $this->db->getUriJson($uri); + foreach ($this->info["provider-includes"] as $provider_uri => $hash) { + if (!isset($packages_info["provider-includes"][$provider_uri]) || + $packages_info["provider-includes"][$provider_uri] != $hash) { $this->updatesAvailableFlag = true; break; } } } - if ($this->updatesAvailableFlag && strlen($__tmp) != file_put_contents($packages_json, $__tmp)) { - throw new \Exception("Couldn't save '$packages_json'"); + if ($this->updatesAvailableFlag && !$this->db->saveUriJson($uri, $this->info)) { + throw new \Exception("Couldn't save '$uri'"); } } @@ -66,45 +81,67 @@ class PickleWeb return $this->updatesAvailableFlag; } - public function fetchProviders() + + protected function saveUriLocal($uri) + { + + } + + protected function getUriLocal($uri) + { + + } + + + protected function diffProviders($remote, $local) { $ret = array(); - /* XXX What meaning does the $hash have? */ - foreach ($this->info["provider-includes"] as $uri => $hash) { - $url = "{$this->host}$uri"; - $__tmp = file_get_contents($url); + foreach ($remote as $vendor => $sha) { + if (isset($local[$vendor]) && $local[$vendor] != $sha) { + $ret[$vendor] = $sha; + } + } - if (!$__tmp) { - //echo "Empty content received from '$url'"; + return $ret; + } + public function fetchProviderUpdates() + { + $ret = array(); + + foreach ($this->info["provider-includes"] as $uri => $hash) { + $pkgs_new = $this->fetchUriJson($uri); + if (!is_array($pkgs_new) || !isset($pkgs_new["providers"]) || !is_array($pkgs_new["providers"]) || empty($pkgs_new["providers"])) { continue; } - $pkgs = json_decode($__tmp, true); + $pkgs = $this->db->getUriJson($uri); if (!is_array($pkgs) || !isset($pkgs["providers"]) || !is_array($pkgs["providers"]) || empty($pkgs["providers"])) { - //echo "No packages provided from '$url'"; + $this->db->saveUriJson($uri, $pkgs_new); + $ret = array_merge($ret, $pkgs_new["providers"]); continue; } - $pkgs = $pkgs["providers"]; - - foreach ($pkgs as $pkg => $phash) { - } + $this->db->saveUriJson($uri, $pkgs_new); - $ret = array_merge($ret, $pkgs); + $ret = array_merge($ret, $this->diffProviders($pkgs_new["providers"], $pkgs["providers"])); } return $ret; } - public function pingBack($data) + public function getNewTags() { - // TODO send the build data to pickle web + $provs = $this->fetchProviderUpdates(); + + + return $provs; } - public function updateDb() + public function pingBack($data) { - + // TODO send the build data to pickle web } + } diff --git a/client/script/pickle_ctl.php b/client/script/pickle_ctl.php index b9ff884..1a27a8c 100644 --- a/client/script/pickle_ctl.php +++ b/client/script/pickle_ctl.php @@ -2,6 +2,7 @@ include __DIR__ . '/../data/config.php'; include __DIR__ . '/../include/PickleWeb.php'; +include __DIR__ . '/../include/PickleDb.php'; include __DIR__ . '/../include/PickleJob.php'; use rmtools as rm; @@ -39,20 +40,20 @@ if ($sync_cmd) { try { - $pw = new rm\PickleWeb($sync_host, $db_dir); + $pw = new rm\PickleWeb($sync_host, new rm\PickleDb($db_dir)); if (!$pw->updatesAvailable()) { echo "No updates available"; exit(0); } - $news = (array)$pw->fetchProviders(); + $news = (array)$pw->getNewTags(); } catch (Exception $e) { echo $e->getMessage() . "\n"; exit(3); } - + var_dump($news); }