refactor(gettask): 1.simple gettask

chromeproxy/background/capture.js

@@ -0,0 +1,32 @@
+
+var href = window.location.href;
+var content = document.documentElement.innerHTML;
+
+setTimeout(function(){
+    window.close();
+}, 15000)
+
+if (condition == undefined) {
+    condition = function () {
+        return true;
+    };
+}
+
+if (condition()) {
+    Tell(BackgroundMsgType.CONTENT, content);
+} else {
+    Tell(BackgroundMsgType.NOTWANT, content);
+}
+
+function Tell(backgroundType, content) {
+    try {
+        if (content == undefined) {
+            chrome.runtime.sendMessage({ type: backgroundType });
+        } else {
+            chrome.runtime.sendMessage({ type: backgroundType, content: content });
+        }
+    } catch (error) {
+        console.log(error);
+        window.location.href = Host;
+    }
+}

chromeproxy/background/config.js

@@ -0,0 +1,23 @@
+// 配置使用的监听
+chrome.webRequest.onBeforeRequest.addListener(function (details) {
+    if (details.url.startsWith("http://eson.config")) {
+      var params = new URL(details.url).searchParams;
+      UpdateHost(params.get("taskurl"))
+      chrome.tabs.remove(details.tabId);
+      chrome.tabs.query({ currentWindow: true }, function (tabs) {
+        var count = 0
+        for (var tab of tabs) {
+          if (!tab.url.startsWith("chrome://")) {
+            count++;
+            chrome.tabs.executeScript(tab.id, { runAt: "document_end", code: `window.location.href = "${Host}"` })
+          } else {
+            chrome.tabs.remove(tab.id);
+          }
+        }
+        for (var i = count; i < WorkerNumber; i++) {
+          chrome.tabs.create({ url: `${Host}` });
+        }
+      })
+      return { cancel: true };
+    }
+  }, { 'urls': ["<all_urls>"] }, ['blocking']);

chromeproxy/background/worker.js

@@ -1,58 +1,81 @@
+console.log("background");
 
 var task_manager = {
 }
 
+chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
+  sendResponse({});
+  switch (request.type) {
+    case BackgroundMsgType.CONTENT:
+      FinishTask(sender, request.content);
+      break;
+    case BackgroundMsgType.ERROR:
+      ErrorTask(sender, request.error)
+      break;
+    case BackgroundMsgType.GETTASK:
+      GetTask(sender);
+      break;
+    case BackgroundMsgType.NOTWANT:
+      ErrorTask(sender, request.content);
+      break;
+    default:
+      break;
+  }
+});
+
 function GetTask(sender) {
   fetch(GetTaskUrl).then(function (response) {
     if (response.ok) {
       response.json().then(function (task) {
-        if(task.code == 200) {
-          task_manager[sender.tab.id] = task;
-          if(task.data.content_condition) {
-            execCode = `PassContentCondition = ${task.data.content_condition};`
-          }
-          chrome.tabs.executeScript(sender.tab.id, {runAt: "document_end", code:  `window.location.href = "${task.data.url}";`}, function (result) {
-            if(execCode) {
-              console.log(execCode);
-              chrome.tabs.executeScript(sender.tab.id, {code: execCode});
-            }
-          })
-          Tell(sender, InjectMsgType.FETCH);
-        } else {
-          Tell(sender, InjectMsgType.NOTASK);
-        }
+        CaptureContent(task);
       })
     } else {
-      // ErrorTask(sender, "error response:" + response.text())
-      console.log("error response:",response.text())
+      console.log("error response:", response.text())
+      ErrorTask(sender, response.text());
     }
   }).catch(function (reason) {
     console.log(reason);
-    // GetTask();
+    ErrorTask(sender, reason);
   })
 }
 
+function CaptureContent(task) {
+  if (task.code == 200) {
+    chrome.tabs.create({ url: task.data.url }, function (tab) {
+      task_manager[tab.id] = task;
+      if (task.data.content_condition) {
+        condition = `${task.data.content_condition}`
+      }
+      if (condition) {
+        chrome.tabs.executeScript(tab.id, { runAt: "document_end", code: condition }, function () {
+          chrome.tabs.executeScript(tab.id, { runAt: "document_end", file: "background/capture.js" });
+        });
+      } else {
+        chrome.tabs.executeScript(tab.id, { runAt: "document_end", file: "background/capture.js" });
+      }
+    });
+  }
+}
+
 function FinishTask(sender, content) {
   var task = task_manager[sender.tab.id];
- 
   var formdata = new FormData();
   formdata.append("taskid", task.data.taskid);
   formdata.append("content", content);
-  // console.log(task.data.taskid);
-  fetch(FinishTaskUrl, {method: "POST", body: formdata }).then(function (response) { 
-    if (response.ok) { 
-      response.json().then(function (value) { 
+  delete task_manager[sender.tab.id];
+  chrome.tabs.remove(sender.tab.id);
+  fetch(FinishTaskUrl, { method: "POST", body: formdata }).then(function (response) {
+    if (response.ok) {
+      response.json().then(function (value) {
         if (value.code == 200) {
           GetTask(sender);
           return
         }
       });
     }
-  }).catch(function(error){
-    // TODO: 汇报错误
-    // ErrorTask(sender, error);
+  }).catch(function (error) {
+    ErrorTask(sender, error);
     console.log(error)
-    // GetTask(sender);
     return;
   });
 }
@@ -62,60 +85,23 @@ function ErrorTask(sender, error) {
   var formdata = new FormData();
   formdata.append("taskid", task.data.taskid);
   formdata.append("error", error);
-  fetch(ErrorTaskUrl, {method: "POST", body:  formdata});
-}
-
-chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
-  sendResponse({});
-  switch (request.type) {
-    case BackgroundMsgType.CONTENT:
-      // 重新获取任务
-      FinishTask(sender, request.content);
-      break;
-    case BackgroundMsgType.ERROR:
-      // 重新获取任务
-      ErrorTask(sender, request.error)
-      GetTask(sender);
-      break;
-    case BackgroundMsgType.GETTASK:
-      GetTask(sender);
-      break;
-    case BackgroundMsgType.NOTWANT:
-      Tell(sender,InjectMsgType.WAIT);
-      break;
-    default:
-      break;
-  }
-});
-
-function Tell(sender, jnjectType) {
-  try {
-    chrome.tabs.sendMessage(sender.tab.id , {type: jnjectType})
-  } catch (error) {
-    console.log(error);
-  }
-}
-
-// 配置使用的监听
-chrome.webRequest.onBeforeRequest.addListener(function (details) {
-  if (details.url.startsWith("http://eson.config")) {
-    var params =  new URL(details.url).searchParams;
-    UpdateHost(params.get("taskurl"))
-    chrome.tabs.remove(details.tabId);
-    chrome.tabs.query({currentWindow: true}, function(tabs) {
-      var count =  0
-      for(var tab of tabs) {
-        if(!tab.url.startsWith("chrome://") ) {
-          count++;
-          chrome.tabs.executeScript(tab.id, {runAt: "document_end", code: `window.location.href = "${Host}"` })
-        } else {
-          chrome.tabs.remove(tab.id);
+  fetch(ErrorTaskUrl, { method: "POST", body: formdata }, function (response) {
+    if (response.ok) {
+      response.json().then(function (value) {
+        if (value.code == 200) {
+          GetTask(sender);
+          return
         }
-      }
-      for(var i = count; i < WorkerNumber;i++) {
-        chrome.tabs.create({url: `${Host}`});
-      }
-    })
-    return { cancel: true };
-  }
-}, { 'urls': ["<all_urls>"] }, ['blocking']);
+      });
+    }
+  }).then().catch(function (error) {
+    console.log(error)
+    return;
+  });
+}
+
+
+ 
+
+
+

chromeproxy/base.js

@@ -5,14 +5,7 @@ var GetTaskUrl = `${Host}/task/get`;
 var FinishTaskUrl = `${Host}/task/content`;
 var ErrorTaskUrl = `${Host}/task/error`;
 
-var WorkerNumber = 10;
-
-var PassContentCondition = undefined;
-// 默认条件所有内容都获取
-var DefaultContentCondition = function () {
-    return true;
-    // return href.startsWith("https://playerduo.com") && content.startsWith('<head></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">')
-}
+var WorkerNumber = 1;
 
 function UpdateHost(host) {
     Host = host 
@@ -30,10 +23,11 @@ const BackgroundMsgType = {
     OK: 'ok',
 };
 
-const InjectMsgType = {
-    FETCH: 'fetch',
-    WAIT: 'wait',
-    ERROR: 'error',
-    NOTASK: 'notask',
-    OK: 'ok',
-};
+// const InjectMsgType = {
+//     FETCH: 'fetch',
+//     WAIT: 'wait',
+//     ERROR: 'error', 
+//     NOTASK: 'notask', // 没有任务
+//     CAPTURE: "capture", // 捕获内容
+//     OK: 'ok',
+// };

chromeproxy/content/inject.js

@@ -1,62 +1,18 @@
-
-
-chrome.runtime.onMessage.addListener(function (request) {
-  switch (request.type) {
-    case InjectMsgType.WAIT:
-      setTimeout(function () {
-        GetTask();
-      }, 6000);
-      break;
-    case InjectMsgType.NOTASK:
-      setTimeout(function () {
-        GetTask();
-      }, 4000);
-      break;
-    case InjectMsgType.FETCH:
-      setTimeout(function () {
-        GetTask();
-      }, 6000)
-      break;
-    default:
-      break;
-  };
-});
-
-
 var href = window.location.href;
-var content = document.documentElement.innerHTML;
-if (href.startsWith(Host)) {
-  GetTask();
-} else {
-  var condition = PassContentCondition != undefined ? PassContentCondition : DefaultContentCondition;
-  if (condition()) {
-    console.log(condition, PassContentCondition);
-    Tell(BackgroundMsgType.CONTENT, content);
-  } else {
-    console.log(condition, PassContentCondition);
-    Tell(BackgroundMsgType.NOTWANT, content);
-  }
-}
-
-
-
-function Tell(backgroundType, content) {
-  try {
-    if (content == undefined) {
-      chrome.runtime.sendMessage({ type: backgroundType });
-    } else {
-      chrome.runtime.sendMessage({ type: backgroundType, content: content });
-    }
-  } catch (error) {
-    console.log(error);
-    window.location.href = Host;
-  }
-
+if(href.startsWith(Host)) {
+  setInterval(function(){
+    GetTask();
+  }, 1);
 }
 
 function GetTask() {
-  Tell(BackgroundMsgType.GETTASK);
-}
+  try {
+    chrome.runtime.sendMessage({ type: BackgroundMsgType.GETTASK }, function(){});
+  } catch (error) {
+    console.log(error);
+  }
+};
+
 
 
 

chromeproxy/manifest.json

@@ -2,10 +2,13 @@
    "background": {
       "scripts": [
          "base.js",
+         "background/config.js",
          "background/worker.js"
       ]
    },
-   "devtools_page": "devtools.html",
+   "web_accessible_resources": [
+      "background/capture.js"
+   ],
    "browser_action": {
       "default_icon": {
          "16": "static/icons/16.png",

proxyserver/callback.go

@@ -29,3 +29,26 @@ func CallbackServer(task *Task) {
 		}
 	}
 }
+
+// ErrorCallbackServer 错误处理回调客户端
+func ErrorCallbackServer(task *Task) {
+	defer func() {
+		if err := recover(); err != nil {
+			log.Println(err)
+		}
+	}()
+
+	task.lock.Lock()
+	defer task.lock.Unlock()
+	if callback, ok := task.data["callback"]; ok {
+		cburl := callback.(string)
+		// log.Println(cburl)
+		_, err := requests.NewSession().Post(cburl).SetBodyAuto(task.data, requests.TypeFormData).Execute()
+		if err != nil {
+			log.Println(err)
+			task.data["error"] = fmt.Sprintf("callback url error: %s", err)
+		} else {
+			task.data["status"] = "readied"
+		}
+	}
+}

proxyserver/router.go

@@ -75,7 +75,6 @@ func PutTask(c *gin.Context) {
 		data.Store("ts", now.UnixNano())
 		data.Store("label", label)
 		data.Store("content_condition", c.PostForm("content_condition"))
-
 		if callback := c.PostForm("callback"); callback != "" {
 			data.Store("callback", callback)
 		}
@@ -165,6 +164,8 @@ func ErrorTask(c *gin.Context) {
 		task.Store("error", errorStr)
 		errorQueue.Push(tid, task) // 进入回调发送队列.TODO: 内容持久化
 		c.JSON(http.StatusOK, Response{Code: 200})
+		// log.Println(errorStr)
+		go ErrorCallbackServer(task)
 		return
 	}
 	c.JSON(http.StatusOK, Response{Code: 404})