TitleCrawlerHandler : Use TitleCrawler.
This commit is contained in:
parent
3b9d6070b6
commit
eaff21a16a
|
@ -5,5 +5,7 @@ package model.crawler
|
||||||
* A crawler that extracts Http Head Title.
|
* A crawler that extracts Http Head Title.
|
||||||
*/
|
*/
|
||||||
class TitleCrawler extends Crawler {
|
class TitleCrawler extends Crawler {
|
||||||
|
override def crawl(url: String): String = {
|
||||||
|
"Mock title."
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,32 +1,52 @@
|
||||||
package com.voronind.doublegis.test
|
package com.voronind.doublegis.test
|
||||||
package model.handler
|
package model.handler
|
||||||
|
|
||||||
import model.lib.StreamUtil
|
import model.lib.{HttpUtil, StreamUtil}
|
||||||
|
|
||||||
import com.sun.net.httpserver.{HttpExchange, HttpHandler}
|
import com.sun.net.httpserver.{HttpExchange, HttpHandler}
|
||||||
import java.io.InputStream
|
import com.voronind.doublegis.test.model.crawler.TitleCrawler
|
||||||
|
|
||||||
|
import java.io.{BufferedReader, ByteArrayInputStream, InputStream, InputStreamReader}
|
||||||
|
import scala.language.postfixOps
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handle Http Title crawling.
|
* Handle Http Title crawling.
|
||||||
*/
|
*/
|
||||||
class TitleCrawlerHandler extends HttpHandler, Handler {
|
class TitleCrawlerHandler extends HttpHandler, Handler {
|
||||||
override def handle(exchange: HttpExchange): Unit = {
|
override def handle(exchange: HttpExchange): Unit = {
|
||||||
log(exchange.getRequestBody)
|
val body = exchange.getRequestBody.readAllBytes()
|
||||||
sendResponse(exchange)
|
log(body)
|
||||||
|
exchange.sendResponse(body)
|
||||||
}
|
}
|
||||||
|
|
||||||
private def log(body: InputStream): Unit = {
|
private def log(body: Array[Byte]): Unit = {
|
||||||
println(f"${TitleCrawlerHandler.CONTEXT}%s : Got payload.")
|
println(f"${TitleCrawlerHandler.CONTEXT}%s : Got payload.")
|
||||||
StreamUtil.copyStream(body, System.out)
|
System.out.write(body)
|
||||||
println()
|
println()
|
||||||
}
|
}
|
||||||
|
|
||||||
private def sendResponse(exchange: HttpExchange): Unit = {
|
// I don't know if this one is ugly, but I wanted to show off a bit.
|
||||||
val response = "Ack!"
|
extension (exchange: HttpExchange) private def sendResponse(request: Array[Byte]): Unit = {
|
||||||
|
val reader = new BufferedReader(new InputStreamReader(ByteArrayInputStream(request)))
|
||||||
|
|
||||||
|
val result = Iterator
|
||||||
|
.continually(reader.readLine)
|
||||||
|
.takeWhile(null !=)
|
||||||
|
.filter(HttpUtil.isUrl)
|
||||||
|
.map({ url => f"$url => ${runCrawler(url)}" })
|
||||||
|
.toList
|
||||||
|
|
||||||
|
val response = result.mkString("\n")
|
||||||
exchange.sendResponseHeaders(200, response.length())
|
exchange.sendResponseHeaders(200, response.length())
|
||||||
|
|
||||||
val out = exchange.getResponseBody
|
val out = exchange.getResponseBody
|
||||||
out.write(response.getBytes)
|
out.write(response.getBytes)
|
||||||
out.close()
|
out.close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private def runCrawler(url: String): String = {
|
||||||
|
new TitleCrawler().crawl(url)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
object TitleCrawlerHandler extends HandlerCompanion {
|
object TitleCrawlerHandler extends HandlerCompanion {
|
||||||
|
|
18
src/main/scala/model/lib/HttpUtil.scala
Normal file
18
src/main/scala/model/lib/HttpUtil.scala
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
package com.voronind.doublegis.test
|
||||||
|
package model.lib
|
||||||
|
|
||||||
|
import java.net.{MalformedURLException, URL}
|
||||||
|
|
||||||
|
object HttpUtil {
|
||||||
|
def isUrl(url: String): Boolean = {
|
||||||
|
try {
|
||||||
|
new URL(url)
|
||||||
|
true
|
||||||
|
} catch {
|
||||||
|
case e: MalformedURLException => {
|
||||||
|
println(f"Got malformed URL: $url%s")
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -13,6 +13,7 @@ class TitleCrawlerServer extends Server {
|
||||||
|
|
||||||
server.start()
|
server.start()
|
||||||
|
|
||||||
|
// TODO: Should have a proper SIGINT handler.
|
||||||
println("Server started! Input any key to exit.")
|
println("Server started! Input any key to exit.")
|
||||||
System.in.read()
|
System.in.read()
|
||||||
server.stop(0)
|
server.stop(0)
|
||||||
|
|
Reference in a new issue