diy了一个爬虫,爬bilibili http2 接口
留下做一个备忘
求解:有没有可以解决长连接 掉线的方法
残缺ORM
<?php
class SingleSql{
private array $connArray = ["host","dbName","user","passwd","dbms"];
private ?PDO $pdo = null;
public string $dsn = "";
private function ArrKey2Str(array $arrkey,string $sep = ",",string $front = "",string $end = "",string $replace = ""){
$res = "";
foreach($arrkey as $key=>$value){
if(strlen($replace) > 0){
$res .= "$front".$replace."$end" . "$sep";
}else{
$res .= "$front".$key."$end" . "$sep";
}
}
$res = str_split($res, strlen($res) - 1)[0];
return $res;
}
private function checkparms(array $array_src,array $array_dst){
foreach($array_src as $key=>$value){
if(in_array($value,array_keys($array_dst))){
continue;
}else{
return false;
}
}
return true;
}
public function __construct(array $cfg){
if($this->checkparms($this->connArray,$cfg)){
$host = $cfg["host"];
$dbName = $cfg["dbName"];
$dbms = $cfg["dbms"];
$dsn="$dbms:host=$host;dbname=$dbName";
$this->dsn = $dsn;
$this->pdo = new PDO($dsn, $cfg["user"], $cfg["passwd"]); //初始化一个PDO对象
}else{
throw new ErrorException("invalid parma of db connection");
}
}
public function Add(string $table,array $data){
$fileds = $this->ArrKey2Str($data);
$binds = $this->ArrKey2Str($data,",",":","");
$sql = "INSERT INTO $table ($fileds) VALUES ($binds)";
$stmt = $this->pdo->prepare($sql);
$arrin = [];
foreach($data as $key=>$value){
$arrin[":$key"] = $value;
}
$stmt->execute($arrin);
}
public function Del(){
}
public function Edit(){
}
public function Fetch(){
}
}
// $cfg = [
// 'dbms' => 'mysql',
// 'host' => '127.0.0.1:3306',
// 'dbName' => 'media',
// 'user' => 'root',
// 'passwd' => 'root',
// ];
// $db = new SingleSql($cfg);
// $db->Add("media.bilibili", [
// "title" => "每次打开门都看到室友*穿衣服怎么办?",
// "av" => 733274169,
// "cover" => "http://i2.hdslb.com/bfs/archive/c956bb70e3489550881c98fac67683780b9084d9.jpg",
// "up_id" => 647256329,
// "up_name" => "白从一wega"
// ]);
<?php
include("./SingleSql.php");
function UnicodeEncode($str){
//split word
preg_match_all('/./u',$str,$matches);
$unicodeStr = "";
foreach($matches[0] as $m){
//拼接
$unicodeStr .= "&#".base_convert(bin2hex(iconv('UTF-8',"UCS-4",$m)),16,10);
}
return $unicodeStr;
}
function HTTP2CURL($url){
$headers = [];
$headers[] = "";
$user_agent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.146 Safari/537.36";
$res = [
"msg"=>"",
"header"=>"",
"body"=>"",
"error"=>""
];
if (
defined("CURL_VERSION_HTTP2") &&
(curl_version()["features"] & CURL_VERSION_HTTP2) !== 0
) {
$ch = curl_init();
curl_setopt_array($ch, [
CURLOPT_URL =>$url,
CURLOPT_HEADER =>true,
CURLOPT_NOBODY =>false,
CURLOPT_RETURNTRANSFER =>true,
CURLOPT_HTTP_VERSION =>CURL_HTTP_VERSION_2_0,
]);
curl_setopt($ch, CURLOPT_HTTPHEADER,$headers);
curl_setopt($ch, CURLOPT_USERAGENT,$user_agent);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0); // self-signed cert
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0); // self-signed cert
$response = curl_exec($ch);
if ($response !== false && strpos($response, "HTTP/2") === 0) {
$headerSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
$header = substr($response, 0, $headerSize);
$body = substr($response, $headerSize, strlen($response)-$headerSize);
$res["body"] = $body;
$res["header"] = $header;
$res["msg"] = "OK";
$res["error"] = "";
curl_close($ch);
return $res;
} elseif ($response !== false) {
curl_close($ch);
$res["msg"] = "ERROR";
$res["error"] = "No HTTP/2 support on server.";
return $res;
} else {
curl_close($ch);
$res["msg"] = "ERROR";
$res["error"] = curl_error($ch);
return $res;
}
} else {
$res["msg"] = "ERROR";
$res["error"] = "No HTTP/2 support on client.";
return $res;
}
}
$cfg = [
'dbms' => 'mysql',
'host' => '127.0.0.1:3306',
'dbName' => 'media',
'user' => 'root',
'passwd' => 'root',
];
$db = new SingleSql($cfg);
while(true){
$url = "https://app.bilibili.com/x/v2/feed/index?ad_extra\u003dE86F4CFF1F8FA890A75155EEAA51E6AE4FA9DBE62FCE708186D0CE5EF37B86940CEF914247F7146520A59A476425C553E2D65C625F41DDFE91804ACCFA2DC1A3A48ED610689D07E83C5134448B644C1AA6266E45E638C6FDBE2C94A7C366E694769CCF470A80B67D40A1750E3F7AB5BC70C36FF966DC586841B3043CC3C93CF0C6F1BC5C80CF1237DD1DA056BAC6CD552DAF7F8F78E4D77FADB30D90A8854DE9C03DBAFA2F59672FDD408EC8AB018F49631F9502BBF2473F8A034480CBECE44020E57E0577DFC7D72D757964209A84AE47F30DAD9A5D4E12C699AF153AEA77292071238AACB7AB8AF8E30EC48E9232998DBB34DFB200BF8AD11CFC6179A1A1690C07B772AFF496666F14E6F74A975D3F246959736FAB881B17ACB06DB8CEC9F903A42C64D1ABC938F5EECABBD9D9AEBCF47B07F4523266CCF7209D90496ABCE2FE3B7C4EADB31C5A903B94366B5CD0AF788504A5A60F28C689D672433B9430248B35F4C70A45141CA7EC5B69DA4250F756211338D9C23AA21106835BC8DBA0FE381D7780AF705DF42A6B875D38811E18998C3BCF296F2261063E02751042386DDEF0CBEA179A03AE752695FCABC3AFAAB56E17B970056B86DF63C356EA1C92797A5DBC9202B88E1D8FC808FF6435B9B844190C9C22FAAD388BACAA58DAB4255E21E9ECF572041802791260040DBC90F23EE98698C3EA7967949678F65913FAF449DCD59CE02C85F70292155A10D87D6F5E782C4C89B90BEAC7A36881DC4A66F92676708399F2BE16CE913D0D64B226DC6F35EC3952FB1B8BABB9E1DDBD5A112C3CC9F0FC7ED70D98D775408F4D39D68EB3DEA21FC87F4AA2AAB9263C9A69D3B4E1DD38AE66E6F2A6AF9F73E68A334CEDD3BEC3D72D76D892C03FC7FFA134F0FCC7A5FEF6DDA3723B1E9E7A73FD570F6F3A59F24FC544FB4803BA8BE519DAA1B7563C5B242E2066BFDA1A699A4FE60144D801544C685020215F9130F59695E7C3277C7C06AB9443E52AB8433EFFF0DC6BE603209E6CFB3AE1E4AF3C444F9EA336\u0026appkey\u003d1d8b6e7d45233436\u0026autoplay_card\u003d11\u0026autoplay_timestamp\u003d0\u0026banner_hash\u003d8444982135872795065\u0026build\u003d7060200\u0026c_locale\u003den_US\u0026channel\u003dyingyongbao\u0026column\u003d2\u0026column_timestamp\u003d0\u0026device_name\u003dRedmi%20K30%20Pro\u0026device_type\u003d0\u0026disable_rcmd\u003d0\u0026flush\u003d6\u0026fnval\u003d976\u0026fnver\u003d0\u0026force_host\u003d0\u0026fourk\u003d1\u0026guidance\u003d0\u0026https_url_req\u003d0\u0026idx\u003d1670065377\u0026inline_danmu\u003d2\u0026inline_sound\u003d1\u0026interest_id\u003d0\u0026login_event\u003d0\u0026mobi_app\u003dandroid\u0026network\u003dmobile\u0026open_event\u003d\u0026platform\u003dandroid\u0026player_net\u003d2\u0026pull\u003dtrue\u0026qn\u003d32\u0026recsys_mode\u003d0\u0026s_locale\u003den_US\u0026splash_id\u003d\u0026statistics\u003d%7B%22appId%22%3A1%2C%22platform%22%3A3%2C%22version%22%3A%227.6.0%22%2C%22abtest%22%3A%22%22%7D\u0026ts\u003d1670068210\u0026video_mode\u003d2\u0026voice_balance\u003d1\u0026sign\u003d243c74ac783e44416f2865562fd74bcf";
$res = json_decode(HTTP2CURL($url)["body"],true);
foreach($res["data"]["items"] as $key=>$value){
$ressd = [];
$ressd["title"] = $value["title"];
$ressd["av"] = $value["param"];
$ressd["cover"] = $value["cover"];
$ressd["up_id"] = $value["args"]["up_id"];
$ressd["up_name"] = $value["args"]["up_name"];
try{
$db->Add("media.bilibili", $ressd);
}catch(Throwable $th){
print($th);
}
print_r($ressd);
echo "\n";
// break;
}
sleep(4);
}