采集組件ZSnoopy · Thinkphp 5.1 筆記

# 1. 說明組件集成Guzzle，可以登錄網站，并下集網站指定鏈接代碼，然后利用phpQuery分析代碼，提取想要信息 ## 采用庫 * 分析庫 1.simple_html_dom 2. phpQuery * 采集庫 1. Guzzle 2. Requests ## 代碼結構 >[info] Zsnoopy.php 實例集成采用QueryList類的簡化操作方法 ``` 代碼結構 Zsnoopy.php -chsys ------| snoopy.php ``` ### Zsnoopy.php ```php <?php namespace app\common\snoopy; use think\Exception; use think\facade\Cache; //加載采集庫 if (!class_exists('Requests')) { // include Env::get('root_path') .'extend/libs/Requests.php'; include app('env')->get('extend_path') . 'libs/Requests.php'; } if (!class_exists('phpQuery')) { // include Env::get('root_path') . 'extend/libs/phpQuery.php'; include app('env')->get('extend_path') . 'libs/phpQuery.php'; } //20170728 添加，用于解決phpQuery解析不了ueeshop升級后分析不了產品詳細信息html問題 /* if(!function_exists('file_get_html')){ include EXTEND_PATH.'/libs/simple_html_dom.php'; } */ \Requests::register_autoloader(); /** * 構建器 * @package app\common\builder * @author 巖 <63453409@qq.com> 集成 https://terryz.oschina.io/ 組件構建快速開發基礎環境 */ class ZSnoopy { //鏈接登錄后對象 protected $conn; protected $cache = true; //默認開啟動緩存 protected $html = null; // protected $log = false; //是否輸出日志，方便觀察程序執行過程 /** * 初始化 * @author 巖 <63453409@qq.com> */ public function __construct() { //不支持小于 tp5.1版本 if (version_compare('5.1', app('app')->version(), '<')) { die('Snoopy is not supert thinkphp version < 5.1 cur thinkphp version is ' . app('app')->version()); // throw new Exception('ZBuilder is not supert thinkphp version > 5.0.11',8001); } /* //加載zbuilder的全局配置 $config = __DIR__.DS.'config.php'; if (is_file($config)) { Config::load($config,'builder'); } */ } //功能完善 /** * 預留自動配置，安裝前端代碼 */ protected function install() { } //動態方法 /** * 創建各種builder的入口 * @param string $type 構建器名稱，'Form', 'Table', 'View' 或其他自定義構建器 * @param string $action 動作 * @author 巖 <63453409@qq.com> * @throws Exception */ public static function engine($type = '', $action = '') { // supper tp5.1 /* if (!defined('DS')) { define('DS', DIRECTORY_SEPARATOR); } */ if ($type == '') { throw new Exception('未指定構建器名稱', 8001); } else { $type = strtolower($type); } //初始化配置 // $config = APP_PATH.'common'.DS.'builder'.DS.$type.DS.'config.php'; /* $config = __DIR__.DS.$type.DS.'config.php'; if (is_file($config)) { Config::load($config,$type); } */ // 構造器類路徑 $class = '\\app\\common\\snoopy\\' . $type . '\\Snoopy'; if (!class_exists($class)) { throw new Exception($type . '構建器不存在', 8002); } return new $class; } /** * 執行任務 * @author 巖 <63453409@qq.com> * @return mixed */ public function run() { } //**********************************************// //**********************************************// //*********輔助處理方法****************// //**********************************************// /* * 攫取后臺數據 */ protected function getUrl($url = null, $cache = true) { $data = $this->cache(md5($url)); if (empty($data) or ($cache == false)) { $result = \Requests::get($url); $this->cache(md5($url), $result->body, 30); $data = $result->body; } return $data; } /* * 驗證鏈接是否正常響應 * @return boolean [true|false] */ public function checkUrl($url) { $curl = curl_init($url); curl_setopt($curl, CURLOPT_NOBODY, true); $result = curl_exec($curl); if ($result !== false) { $statusCode = curl_getinfo($curl, CURLINFO_HTTP_CODE); if ($statusCode == 200) { return true; } else { return false; } } else { return false; } } public function put($mess) { echo '[' . date('Y-m-d H:i:m') . ']' . $mess . PHP_EOL; } /* * 緩存數據 * @param string $key 緩存鍵名稱 * @param [array|string] $data 緩存值 * @resturn [array|string|int|float] $result 返回數據 */ public function cache($key = null, $data = null, $timeout = 3600) { //如$key為默認值，則表示無設置，執行清空緩存 if (empty($key)) { Cache::clear('Zsnoopy'); return true; } //開啟緩存則 if ($this->cache) { if (isset($data)) { Cache::tag('Zsnoopy')->set('snoopy_' . (string) $key, $data, $timeout); $result = $data; } $result = Cache::tag('Zsnoopy')->get('snoopy_' . $key); } return isset($result) ? $result : $data; } } ``` ### snoopy.php ```php <?php namespace app\common\snoopy\chsys; use app\common\snoopy\ZSnoopy; class Snoopy extends ZSnoopy { protected $_cookie = null; //保存登錄系統后的cookie認證信息 protected $cache = true; //默認啟動 cache protected $log = false; protected $config = [ ]; //默認需要登錄系統 public function __construct(array $loginConf = []) //初始化登錄后臺 { } //配置是否使用緩存 public function setCache($value = false) { $this->cache = $value; return $this; } //驗證mrtg設備鏈接地址是否正確 public function checkMrtgUrl($url) { \phpQuery::newDocumentHTML($this->getUrl($url, false)); $setName = 'chsysname'; // $setName = pq('body > h1')->text(); return trim($setName); } /* * 獲取設備所有端口鏈接信息 */ public function getSetPost($url, $setName) { \phpQuery::newDocumentHTML($this->getUrl($url, false)); $list = []; // $tds = pq('body > table:nth-child(2)')->find('td'); // dump($tds); foreach ($tds as $_td) { $_tmp = []; $name = pq($_td)->find('b')->text(); if (preg_match('/([\w\/\.]+) -- ([\w\/\-]+)/i', $name, $names)) { // trace($names,'log'); $_tmp['setname'] = $setName; //數據表中，需要這個稱名 $_tmp['portname'] = $names[1]; $_tmp['hostname'] = $names[2]; $_tmp['url'] = pq($_td)->find('a')->attr('href'); $picUrl = pq($_td)->find('img')->attr('src'); $_tmp['picUrl'] = empty($picUrl) ? null : $url . pq($_td)->find('img')->attr('src'); $list[] = $_tmp; } } return $list; } //獲取MRTG鏈接所有圖片鏈接地址 public function getPics($url) { $baseUrl = pathinfo($url)['dirname']; // trace($baseUrl,'log'); \phpQuery::newDocumentHTML($this->getUrl($url, false)); $list = []; // $tds = pq('.graph'); // dump($tds); foreach ($tds as $_td) { $_tmp = []; $picName = pq($_td)->find('img')->attr('src'); $_tmp['picName'] = $picName; $_tmp['picUrl'] = empty($picName) ? null : $baseUrl . '/' . $picName; $list[] = $_tmp; } return $list; } public function viewSet($url) { return $this->getUrl($url, false); } public function getPost($url) { return $this->getUrl($url, false); } //通過url下載圖片 public static function getUrlToPic($url, $fileName, $cacheTime = 300) { self::checkFile($fileName, $cacheTime); if (!file_exists($fileName)) { file_put_contents($fileName, file_get_contents($url)); } } //判斷文件保存時間 /* * @param string $fileName 文件全路徑名 * @param int $cacheTime 緩存時間，默認300秒，則5分鐘 * @return boolean [true|false] 文件是否符合 */ public static function checkFile($fileName, $cacheTime = 300) { if (file_exists($fileName)) { if ((time() - filectime($fileName)) > $cacheTime) { unlink($fileName); } } } } ``` # 2.應用實例