Ruby's URI library provides comprehensive tools for parsing, manipulating, and working with Uniform Resource Identifiers (URIs). It's essential for web scraping, API integration, and URL manipulation.
require 'uri'
# Parse a complete URL
uri = URI.parse('https://api.example.com:8080/v1/users?active=true&role=admin#section1')
puts uri.scheme # => "https"
puts uri.host # => "api.example.com"
puts uri.port # => 8080
puts uri.path # => "/v1/users"
puts uri.query # => "active=true&role=admin"
puts uri.fragment # => "section1"
puts uri.to_s # => Full URLrequire 'uri'
# Build URI from components
uri = URI::HTTPS.build(
host: 'api.github.com',
path: '/repos/ruby/ruby',
query: 'per_page=50&sort=updated'
)
puts uri.to_s
# => "https://api.github.com/repos/ruby/ruby?per_page=50&sort=updated"
# Using URI.build
uri2 = URI.build(
scheme: 'https',
host: 'example.com',
port: 443,
path: '/api/v1/data',
query: URI.encode_www_form(limit: 100, offset: 0)
)
puts uri2.to_srequire 'uri'
# Parse query parameters
uri = URI.parse('https://example.com/search?q=ruby&type=repo&sort=stars')
params = URI.decode_www_form(uri.query)
puts params # => [["q", "ruby"], ["type", "repo"], ["sort", "stars"]]
# Convert to hash
param_hash = params.to_h
puts param_hash # => {"q"=>"ruby", "type"=>"repo", "sort"=>"stars"}
# Build query string from hash
new_params = {
query: 'ruby programming',
category: 'tutorials',
limit: 10
}
query_string = URI.encode_www_form(new_params)
puts query_string # => "query=ruby+programming&category=tutorials&limit=10"
# Create new URI with query
base_uri = URI.parse('https://api.example.com/search')
base_uri.query = query_string
puts base_uri.to_sThe open-uri library extends the standard library to make opening URLs as simple as opening files. It's perfect for quick web scraping and data fetching tasks.
require 'open-uri'
# Simple URL opening
content = URI.open('https://httpbin.org/json').read
puts content
# Opening with block (automatic closing)
URI.open('https://httpbin.org/user-agent') do |response|
puts response.read
puts "Status: #{response.status}"
puts "Content-Type: #{response.content_type}"
endrequire 'open-uri'
# Custom headers
options = {
'User-Agent' => 'Mozilla/5.0 (Ruby Script)',
'Accept' => 'application/json',
'Authorization' => 'Bearer your-token-here'
}
begin
response = URI.open('https://api.github.com/user', options)
data = response.read
puts "Response: #{data}"
puts "Content-Type: #{response.content_type}"
puts "Last-Modified: #{response.last_modified}"
rescue OpenURI::HTTPError => e
puts "HTTP Error: #{e.message}"
endrequire 'open-uri'
require 'json'
# JSON API response
def fetch_json(url)
response = URI.open(url, 'Accept' => 'application/json')
JSON.parse(response.read)
rescue JSON::ParserError => e
puts "JSON parsing error: #{e.message}"
nil
rescue OpenURI::HTTPError => e
puts "HTTP error: #{e.message}"
nil
end
# Example usage
weather_data = fetch_json('https://httpbin.org/json')
puts weather_data
# Download binary files
def download_file(url, filename)
URI.open(url, 'rb') do |response|
File.open(filename, 'wb') do |file|
file.write(response.read)
end
end
puts "Downloaded #{filename}"
rescue => e
puts "Download failed: #{e.message}"
end
# Download an image
download_file('https://httpbin.org/image/png', 'test_image.png')Combine URI parsing with HTML parsing libraries like Nokogiri for effective web scraping.
require 'open-uri'
require 'nokogiri'
class WebScraper
def initialize(base_url)
@base_url = base_url
end
def scrape_page(path)
url = URI.join(@base_url, path)
puts "Scraping: #{url}"
doc = Nokogiri::HTML(URI.open(url))
{
title: doc.css('title').text.strip,
headings: doc.css('h1, h2, h3').map(&:text),
links: extract_links(doc),
meta_description: doc.css('meta[name="description"]').first&.[]('content')
}
rescue => e
puts "Error scraping #{url}: #{e.message}"
nil
end
private
def extract_links(doc)
doc.css('a[href]').map do |link|
href = link['href']
next if href.nil? || href.empty?
{
text: link.text.strip,
url: resolve_url(href),
external: external_link?(href)
}
end.compact
end
def resolve_url(href)
uri = URI.parse(href)
return href if uri.absolute?
URI.join(@base_url, href).to_s
rescue URI::InvalidURIError
nil
end
def external_link?(href)
uri = URI.parse(href)
return false unless uri.absolute?
base_uri = URI.parse(@base_url)
uri.host != base_uri.host
rescue URI::InvalidURIError
false
end
end
# Example usage
scraper = WebScraper.new('https://example.com')
page_data = scraper.scrape_page('/about')
puts page_datarequire 'open-uri'
require 'nokogiri'
class RateLimitedScraper
def initialize(delay: 1.0, max_retries: 3)
@delay = delay
@max_retries = max_retries
@last_request_time = 0
end
def fetch_with_retry(url, headers = {})
retries = 0
begin
# Rate limiting
sleep_time = @delay - (Time.now - @last_request_time)
sleep(sleep_time) if sleep_time > 0
@last_request_time = Time.now
# Make request
response = URI.open(url, headers)
{
success: true,
content: response.read,
headers: response.meta
}
rescue OpenURI::HTTPError => e
retries += 1
if retries <= @max_retries && should_retry?(e)
puts "Retrying #{url} (attempt #{retries})"
sleep(@delay * retries) # Exponential backoff
retry
else
{ success: false, error: e.message }
end
rescue => e
{ success: false, error: e.message }
end
end
def scrape_links(start_url, max_pages: 10)
visited = Set.new
to_visit = [start_url]
results = []
while to_visit.any? && results.length < max_pages
url = to_visit.shift
next if visited.include?(url)
visited.add(url)
puts "Scraping: #{url}"
result = fetch_with_retry(url)
next unless result[:success]
doc = Nokogiri::HTML(result[:content])
page_info = {
url: url,
title: doc.css('title').text.strip,
links: []
}
# Extract internal links
doc.css('a[href]').each do |link|
href = link['href']
next unless href
absolute_url = resolve_url(url, href)
next unless absolute_url && same_domain?(start_url, absolute_url)
page_info[:links] << {
text: link.text.strip,
url: absolute_url
}
# Add to visit queue if not visited
to_visit << absolute_url unless visited.include?(absolute_url)
end
results << page_info
end
results
end
private
def should_retry?(error)
# Retry on server errors but not client errors
error.message.include?('5') || error.message.include?('429')
end
def resolve_url(base_url, relative_url)
URI.join(base_url, relative_url).to_s
rescue URI::InvalidURIError
nil
end
def same_domain?(url1, url2)
URI.parse(url1).host == URI.parse(url2).host
rescue URI::InvalidURIError
false
end
end
# Example usage
scraper = RateLimitedScraper.new(delay: 2.0)
results = scraper.scrape_links('https://example.com', max_pages: 5)
puts "Scraped #{results.length} pages"require 'uri'
class URLBuilder
def initialize(base_url)
@uri = URI.parse(base_url)
end
def path(new_path)
@uri.path = new_path.start_with?('/') ? new_path : "/#{new_path}"
self
end
def add_path(segment)
current_path = @uri.path.end_with?('/') ? @uri.path : "#{@uri.path}/"
@uri.path = "#{current_path}#{segment}"
self
end
def query(params = {})
existing_params = @uri.query ? URI.decode_www_form(@uri.query).to_h : {}
merged_params = existing_params.merge(params.transform_keys(&:to_s))
@uri.query = URI.encode_www_form(merged_params)
self
end
def remove_query(key)
return self unless @uri.query
params = URI.decode_www_form(@uri.query).to_h
params.delete(key.to_s)
@uri.query = params.empty? ? nil : URI.encode_www_form(params)
self
end
def fragment(frag)
@uri.fragment = frag
self
end
def secure
@uri.scheme = 'https'
@uri.port = 443 if @uri.port == 80
self
end
def port(new_port)
@uri.port = new_port
self
end
def to_s
@uri.to_s
end
def to_uri
@uri.dup
end
end
# Example usage
url = URLBuilder.new('http://api.example.com')
.secure
.path('/v2/users')
.query(active: true, limit: 50, sort: 'created_at')
.fragment('results')
.to_s
puts url
# => "https://api.example.com/v2/users?active=true&limit=50&sort=created_at#results"require 'uri'
class URLValidator
ALLOWED_SCHEMES = %w[http https ftp].freeze
def self.valid?(url)
uri = URI.parse(url)
uri.kind_of?(URI::HTTP) || uri.kind_of?(URI::HTTPS) || uri.kind_of?(URI::FTP)
rescue URI::InvalidURIError
false
end
def self.safe?(url)
return false unless valid?(url)
uri = URI.parse(url)
# Check scheme
return false unless ALLOWED_SCHEMES.include?(uri.scheme.downcase)
# Check for dangerous characters
return false if url.include?('..') || url.include?('<') || url.include?('>')
# Check host is not localhost or private IP
return false if localhost_or_private?(uri.host)
true
rescue
false
end
def self.sanitize(url)
return nil unless valid?(url)
uri = URI.parse(url)
# Remove dangerous fragments
uri.fragment = nil if uri.fragment&.include?('<')
# Encode spaces and special characters in path
if uri.path
uri.path = URI.encode_www_form_component(URI.decode_www_form_component(uri.path))
end
# Clean query parameters
if uri.query
params = URI.decode_www_form(uri.query)
clean_params = params.reject { |k, v| k.include?('<') || v.include?('>') }
uri.query = clean_params.empty? ? nil : URI.encode_www_form(clean_params)
end
uri.to_s
rescue
nil
end
private
def self.localhost_or_private?(host)
return true if host == 'localhost' || host == '127.0.0.1'
# Check for private IP ranges
return true if host.match?(/^10\./)
return true if host.match?(/^192\.168\./)
return true if host.match?(/^172\.(1[6-9]|2[0-9]|3[01])\./)
false
end
end
# Example usage
urls = [
'https://example.com/safe',
'javascript:alert("xss")',
'https://localhost/admin',
'https://example.com/path with spaces',
'https://example.com/search?q=<script>alert("xss")</script>'
]
urls.each do |url|
puts "#{url}:"
puts " Valid: #{URLValidator.valid?(url)}"
puts " Safe: #{URLValidator.safe?(url)}"
puts " Sanitized: #{URLValidator.sanitize(url)}"
puts
endrequire 'uri'
require 'net/http'
require 'json'
class HTTPClient
def initialize(base_url, timeout: 30)
@base_uri = URI.parse(base_url)
@timeout = timeout
@default_headers = {
'User-Agent' => 'Ruby HTTP Client',
'Accept' => 'application/json',
'Content-Type' => 'application/json'
}
end
def get(path, params: {}, headers: {})
uri = build_uri(path, params)
request = Net::HTTP::Get.new(uri)
execute_request(uri, request, headers)
end
def post(path, data: {}, headers: {})
uri = build_uri(path)
request = Net::HTTP::Post.new(uri)
request.body = data.is_a?(String) ? data : data.to_json
execute_request(uri, request, headers)
end
def put(path, data: {}, headers: {})
uri = build_uri(path)
request = Net::HTTP::Put.new(uri)
request.body = data.is_a?(String) ? data : data.to_json
execute_request(uri, request, headers)
end
def delete(path, headers: {})
uri = build_uri(path)
request = Net::HTTP::Delete.new(uri)
execute_request(uri, request, headers)
end
private
def build_uri(path, params = {})
uri = @base_uri.dup
uri.path = File.join(uri.path, path)
unless params.empty?
uri.query = URI.encode_www_form(params)
end
uri
end
def execute_request(uri, request, headers)
# Merge headers
(@default_headers.merge(headers)).each do |key, value|
request[key] = value
end
# Execute request
Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
http.read_timeout = @timeout
response = http.request(request)
{
status: response.code.to_i,
headers: response.to_hash,
body: response.body,
success: response.code.to_i.between?(200, 299)
}
end
rescue => e
{
status: 0,
headers: {},
body: nil,
success: false,
error: e.message
}
end
end
# Example usage
client = HTTPClient.new('https://jsonplaceholder.typicode.com')
# GET request with parameters
response = client.get('/posts', params: { userId: 1 })
puts "GET Response: #{response[:status]}"
# POST request
new_post = {
title: 'Test Post',
body: 'This is a test post',
userId: 1
}
response = client.post('/posts', data: new_post)
puts "POST Response: #{response[:status]}"require 'uri'
require 'open-uri'
require 'timeout'
class RobustURLHandler
def self.fetch_with_fallback(urls, timeout: 10)
urls = Array(urls)
urls.each_with_index do |url, index|
begin
puts "Trying URL #{index + 1}: #{url}"
result = Timeout.timeout(timeout) do
URI.open(url) do |response|
{
url: url,
content: response.read,
status: response.status,
content_type: response.content_type,
success: true
}
end
end
return result
rescue Timeout::Error
puts "Timeout for #{url}"
next
rescue OpenURI::HTTPError => e
puts "HTTP error for #{url}: #{e.message}"
next
rescue => e
puts "Error for #{url}: #{e.message}"
next
end
end
{
url: nil,
content: nil,
status: nil,
content_type: nil,
success: false,
error: 'All URLs failed'
}
end
def self.extract_urls_from_text(text)
# Regex to match URLs
url_regex = /https?:\/\/(?:www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b(?:[-a-zA-Z0-9()@:%_\+.~#?&=]*)/
urls = text.scan(url_regex)
# Validate and return only valid URLs
urls.select { |url| URLValidator.valid?(url) }
end
def self.follow_redirects(url, max_redirects: 5)
redirects = 0
current_url = url
loop do
uri = URI.parse(current_url)
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
request = Net::HTTP::Get.new(uri)
response = http.request(request)
case response
when Net::HTTPSuccess
return {
final_url: current_url,
redirects: redirects,
response: response
}
when Net::HTTPRedirection
redirects += 1
if redirects > max_redirects
raise "Too many redirects (#{max_redirects})"
end
location = response['location']
current_url = URI.join(current_url, location).to_s
puts "Redirect #{redirects}: #{current_url}"
else
raise "HTTP Error: #{response.code} #{response.message}"
end
end
end
rescue => e
{
final_url: current_url,
redirects: redirects,
error: e.message
}
end
end
# Example usage
urls = [
'https://httpbin.org/status/500', # This will fail
'https://httpbin.org/json', # This should work
'https://httpbin.org/delay/1' # Backup URL
]
result = RobustURLHandler.fetch_with_fallback(urls)
puts "Final result: #{result[:success] ? 'Success' : 'Failed'}"
# Extract URLs from text
text = "Check out https://github.com and https://stackoverflow.com for help"
found_urls = RobustURLHandler.extract_urls_from_text(text)
puts "Found URLs: #{found_urls}"Watch and learn uri & opening urls