require "open-uri"

# 直接
html = open(link).read
# 使用 User-Agent
html = open(link, "User-Agent" => "Mozilla/5.0 ...").read
# 使用 代理
html = open(link, "proxy" => "http://127.0.0.1:8001").read
# 如果有编码问题, 用二进制方式
html = open(link, "r:binary").read.encode("utf-8", uri.charset, invalid: :replace, undef: :replace)

require "socksify/http"

html = Net::HTTP::SOCKSProxy("127.0.0.1", 8002).get(URI(link))

# !!! Copy from: https://github.com/sparklemotion/nokogiri#synopsis

require 'nokogiri'
require 'open-uri'

# Fetch and parse HTML document
doc = Nokogiri::HTML(open('http://www.nokogiri.org/tutorials/installing_nokogiri.html'))

puts "### Search for nodes by css"
doc.css('nav ul.menu li a', 'article h2').each do |link|
  puts link.content
end

puts "### Search for nodes by xpath"
doc.xpath('//nav//ul//li/a', '//article//h2').each do |link|
  puts link.content
end

puts "### Or mix and match."
doc.search('nav ul.menu li a', '//article//h2').each do |link|
  puts link.content
end