Tips and tricks for building api heavy ruby on rails applications

Tips and Tricks for
Building API-Heavy Ruby
on Rails Applications

Tim Cull

@trcull
trcull@pollen.io

We

We've Learned Some Things

Authentication Still Hurts

image: http://wiki.oauth.net/

Do Everything in Background
Tasks

Y T
O H
U R
O
W T
I T
L L
L E
D
B
E

Libraries
Pull In Problems

http = Net::HTTP.new "api.cafepress.com", 80

http.set_debug_output STDOUT

http = Net::HTTP.new "api.cafepress.com", 80

http.set_debug_output STDOUT

<- "POST /oauth/AccessToken HTTP/1.1rnAccept: */*rnUser-Agent: OAuth gem v0.4.7rnContent-
Length: 0rnAuthorization: OAuth oauth_body_hash="2jmj7YBwk%3D", oauth_callback="http%3A%2F%
2Flocalhost%3A3000%2Fxero%2Foauth_callback", oauth_consumer_key="20Q0CD6", oauth_nonce="
BawDBGd0kRDdCM", oauth_signature="B7Bd%3D", oauth_signature_method="RSA-SHA1",
oauth_timestamp="1358966217", oauth_token="MyString", oauth_version="1.0"rnConnection:
closernHost: api-partner.network.xero.comrnrn"
-> "HTTP/1.1 401 Unauthorizedrn"
-> "Cache-Control: privatern"
-> "Content-Type: text/html; charset=utf-8rn"
-> "Server: Microsoft-IIS/7.0rn"
-> "X-AspNetMvc-Version: 2.0rn"
-> "WWW-Authenticate: OAuth Realm="108.254.144.237"rn"
-> "X-AspNet-Version: 4.0.30319rn"
-> "X-S: api1rn"
-> "Content-Length: 121rn"
-> "rn"
reading 121 bytes...
-> "oauth_problem=token_rejected&oauth_problem_advice=Token%20MyString%20does%20not%
20match%20an%20expected%20REQUEST%20token"

oauth = OAuth::Consumer.new( key, secret)

oauth.http.set_debug_output STDOUT

oauth = OAuth::Consumer.new( key, secret)

oauth = OAuth::Consumer.new( key, secret, {
:request_token_url => 'https://api.linkedin.
com/uas/oauth/requestToken'})


# NOTHING PRINTS!

:request_token_url => 'https://api.linkedin.
com/uas/oauth/requestToken'})

# NOTHING PRINTS!
OAuth::Consumer.rb (151)
def request(http_method, path, token = nil, *arguments)
if path !~ /^//
@http = create_http(path)
_uri = URI.parse(path)
end
.....
end

module OAuth
class Consumer
def create_http_with_featureviz(*args)
@http ||= create_http_without_featureviz(*args).tap do |http|
begin
http.set_debug_output($stdout) unless options[:
suppress_debugging]
rescue => e
puts "error trying to set debugging #{e.message}"
end
end
end
alias_method_chain :create_http, :featureviz
end
end

Class Structure

Make an API class

Responsibilities
● Network plumbing
● Translating xml/json/whatever to "smart"
hashes *
● Converting Java-like or XML-like field
names to Ruby-like field names
● Parsing string dates and integers and
turning them into real Ruby data types
● Setting reasonable defaults
● Flattening awkward data structures
● That’s it!

172 Lines of Code
https://github.com/trcull/pollen-snippets/blob/master/lib/abstract_api.rb

class Instagram::Api
def get_photos()
resp = get("v1/users/#{@user_id}/media/recent")
parse_response_and_stuff resp
end

def get(url)
req = Net::HTTP::Get.new url
do_request req
end

def do_request( req )
net = Net::HTTP.new
net.start do |http|
http.request req
end
end
end

Goal

api = InstagramApi.new current_user

photos = api.get_photos

photos.each do |photo|
puts photo.thumbnail_url
end

def get_photos
rv = []
response = get( "v1/users/#{@user_id}/media/recent",
{:access_token=>@access_token})
data = JSON.parse(response.body)
data['data'].each do |image|
photo = ApiResult.new
photo[:id] = image['id']
photo[:thumbnail_url] = image['images']['thumbnail']['url']
if image['caption'].present?
photo[:caption] = image['caption']['text']
else
photo[:caption] = 'Instagram image'
end
rv << photo
end
rv
end

def get(url, params, with_retry=true)

real_url = "#{@protocol}://#{@host}/#{url}?"
.concat(params.collect{|k,v|
"#{k}=#{CGI::escape(v.to_s)}"}.join("&"))

request = Net::HTTP::Get.new(real_url)

if with_retry
response = do_request_with_retry request
else
response = do_request request
end

response

end

class ApiResult < Hash
def []=(key, value)
store(key.to_sym,value)
methodify_key key.to_sym
end

def methodify_key(key)
if !self.respond_to? key
self.class.send(:define_method, key) do
return self[key]
end
self.class.send(:define_method, "#{key}=") do |val|
self[key] = val
end
end
end
end
See: https://github.com/trcull/pollen-snippets/blob/master/lib/api_result.rb

describe Instagram::Api do
subject do
user = create(:user, :token=>token, :secret=>secret)
Instagram::Api.new(user)
end

describe "making fake HTTP calls" do

end

describe "in a test bed" do

end

describe "making real HTTP calls" , :integration => true do

end
end

in lib/tasks/functional.rake

RSpec::Core::RakeTask.new("spec:integration") do |t|
t.name = "spec:integration"
t.pattern = "./spec/**/*_spec.rb"
t.rspec_opts = "--tag integration --profile"
end

in .rspec

--tag ~integration --profile

to run: "rake spec:integration"

subject do
#sometimes have to be gotten manually, unfortunately.
user = create(:user,
:token=>"stuff",
:secret=>"more stuff")
end

describe "in a test bed" do
it "pulls the caption out of photos" do
photos = subject.get_photos
photos[0].caption.should eq 'My Test Photo'
end
end

end

subject do
#sometimes have to be gotten manually, unfortunately.
user = create(:user,
:token=>"stuff",
:secret=>"more stuff")
end

describe making real HTTP calls" , :integration => true do
it "pulls the caption out of photos" do
photos[0].caption.should eq 'My Test Photo'
end
end

end

describe Evernote::Api do
describe "making real HTTP calls" , :integration => true do
subject do
req_token = Evernote::Api.oauth_request_token
oauth_verifier = Evernote::Api.authorize_as('testacct','testpass', req_token)
access_token = Evernote::Api.oauth_access_token(req_token,
oauth_verifier)
Evernote::Api.new access_token.token, access_token.secret
end

it "can get note" do
note_guid = "4fb9889d-813a-4fa5-b32a-1d3fe5f102b3"
note = subject.get_note note_guid
note.guid.should == note_guid
end
end
end

def authorize_as(user, password, request_token)
#pretend like the user logged in
get("Login.action") #force a session to start
session_id = @cookies['JSESSIONID'].split('=')[1]
login_response = post("Login.action;jsessionid=#{session_id}",
{:username=>user,
:password=>password,
:login=>'Sign In',
:targetUrl=>CGI.escape("/OAuth.action?oauth_token=#{request_token.token}")})

response = post('OAuth.action', {:authorize=>"Authorize",
:oauth_token=>request_token.token})

location = response['location'].scan(/oauth_verifier=(d*w*)/)

oauth_verifier = location[0][0]
oauth_verifier
end

describe "making fake HTTP calls" do
before do
Net::HTTP.stub(:new).and_raise("unexpected network call")
end

it "pulls the thumbnail out of photos" do
response = double("response")
data_hash = {"data" =>
[{
"link"=>"apple.jpg",
"id"=>"12",
"images"=>
{
"thumbnail"=>{"url"=>"www12"},
"standard_resolution"=>{"url"=>"www12"}
}
}]}
response.stub(:body).and_return data_hash.to_json
subject.should_receive(:do_request).and_return(response)
photos[0].thumbnail_url.should eq 'www12'
end
end

describe EvernoteController do
before do
@api = double('fakeapi')
Evernote::Api.stub(:new).and_return @api
end

describe "list_notes" do
it "should list notes by title" do
a_note = ApiResult.new({:title=>'test title'})
@api.should_receive(:get_notes).and_return [a_note]
get "/mynotes"
response.body.should match /<td>test title</td>/
end
end
end

Ask for a Request Token

Redirect User to Site (w/ Request Token)

User Logs in and Authorizes

Site Redirects Back to You W/ OAuth Verifier

Trade OAuth Verifier and Request Token for
an Access Token

Store Access Token (Securely)

Make API Calls W/ Access Token

Vanilla
@consumer = OAuth::Consumer.new("key","secret", :site => "https://agree2")

@callback_url = "http://127.0.0.1:3000/oauth/callback"
@request_token = @consumer
.get_request_token(:oauth_callback => @callback_url)
session[:request_token] = @request_token

redirect_to @request_token.authorize_url(:oauth_callback => @callback_url)

#user is on other site, wait for callback

@access_token = session[:request_token]
.get_access_token(:oauth_verifier=>params[:verifier])

@photos = @access_token.get('/photos.xml')

Source: https://github.com/oauth/oauth-ruby

Instagram
#redirect user first, no request token necessary
@redirect_uri = CGI.escape('http:///myapp.com/oauth/callback')
redirect_to "https://api.instagram.com/oauth/authorize
?client_id=xyz123
&redirect_uri=#{@redirect_uri}
&response_type=code
&scope=comments+basic"

#wait for callback

response = post( "oauth/access_token", {:client_id=>@client_id,
:client_secret=>@client_secret,
:redirect_uri=>@redirect_uri,
:grant_type=>'authorization_code',
:code=>params[:code]})
json = JSON.parse response.body

access_token = json['access_token']
instagram_name = json['user']['username']
instagram_id = json['user']['id']

Freshbooks
require 'oauth'
require 'oauth/signature/plaintext'

:scheme=> :query_string,
:signature_method=>"PLAINTEXT",
:oauth_callback=>callback,
:authorize_path => "/oauth/oauth_authorize.php",
:access_token_path=>"/oauth/oauth_access.php",
:request_token_path=>"/oauth/oauth_request.php",
:site=>"http://#{usersitename}.freshbooks.com"})

Xero
@consumer = OAuth::Consumer.new(@oauth_key, @oauth_secret,{
:site => "https://api-partner.network.xero.com:443",
:signature_method => 'RSA-SHA1',
:private_key_str => ENV['XERO_CLIENT_PEM'] ,
:ssl_client_cert=>ENV['XERO_ENTRUST_SSL_CERT'],
:ssl_client_key=>ENV['XERO_ENTRUST_PRIVATE_PEM']})

module OAuth
class Consumer
def create_http_with_featureviz(*args)
@http ||= create_http_without_featureviz(*args).tap do |http|
http.cert = OpenSSL::X509::Certificate.new(options[:ssl_client_cert]) if
options[:ssl_client_cert]
http.key = OpenSSL::PKey::RSA.new( options[:ssl_client_key]) if options[:
ssl_client_key]
end
end
alias_method_chain :create_http, :featureviz
end
end Thank you @tlconnor

@consumer = OAuth.new(@oauth_key, @oauth_secret,{
:site => "https://api-partner.network.xero.com:443",
:signature_method => 'RSA-SHA1',
:private_key_str => ENV['XERO_CLIENT_PEM'] ,
:ssl_client_cert=>ENV['XERO_ENTRUST_SSL_CERT'],
:ssl_client_key=>ENV['XERO_ENTRUST_PRIVATE_PEM']})

module OAuth::Signature::RSA
class SHA1 < OAuth::Signature::Base
def digest
private_key = OpenSSL::PKey::RSA.new(
if options[:private_key_str]
options[:private_key_str]
elsif options[:private_key_file]
IO.read(options[:private_key_file])
else
consumer_secret
end
)
private_key.sign(OpenSSL::Digest::SHA1.new, signature_base_string)
end
end
end

Evernote
consumer = OAuth::Consumer.new(key, secret, {
:site => "https://www.evernote.com",
:authorize_path => "/OAuth.action"})

# redirect, yadda, yadda wait for callback

access_token = request_token.get_access_token(:oauth_verifier => oauth_verifier)

note_store_url = access_token.params['edam_noteStoreUrl']

#This is Thrift
transport = Thrift::HTTPClientTransport.new note_store_url
protocol = Thrift::BinaryProtocol.new transport
store = Evernote::EDAM::NoteStore::NoteStore::Client.new protocol

#Yay! Finally Notebooks!
notebooks = store.listNotebooks access_token.token

Background
Processing

i.e. Do It

class SlurpImagesJob
def self.enqueue(user_id)
SlurpImagesJob.new.delay.perform(user_id)
end

#this makes unit testing with simulated errors easier
def perform(user_id)
begin
do_perform user_id
rescue => e
Alerts.log_error "We encountered an error slurping your Instagram images please try again",
e
end
end

def do_perform(user_id)
user = User.find user_id
cafepress = Cafepress::Api.new user
instagram = Instagram::Api.new user

photos = instagram.get_photos
photos.each do |photo|
cafepress.upload_design photo.caption, photo.standard_resolution_url
end
end
end

class InstagramController < ApplicationController

def oauth_callback
request_access_token params
SlurpImagesJob.enqueue current_user.id
end
end

describe InstagramController do
it "enqueues a job to slurp images" do
SlurpImagesJob.should_receive :enqueue
post '/oauth_callback'
end
end

Register a Callback URL for a User

Site Verifies URL actually works (typically)

User Does Something

Site Calls Back URL With an ID (typically)

Application Polls Site for More Details

Application Does Whatever

class FreshbooksApi < AbstractApi
def register_callback(user_id)
xml = "<request method="callback.create">
<callback>
<event>invoice.create</event>
<uri>http://app.featureviz.com/webhook/freshbooks/#{user_id}</uri>
</callback>
</request>"
post_with_body("api/2.1/xml-in", xml)
end

def verify_callback(our_user_id, verifier, callback_id)
xml = "<request method="callback.verify">
<callback>
<callback_id>#{callback_id}</callback_id>
<verifier>#{verifier}</verifier>
</callback>
</request>"
post_with_body("api/2.1/xml-in", xml)
end
end

class WebhooksController < ApplicationController
# URL would be /webhook/freshbooks/:our_user_id
def freshbooks_callback
our_user_id = params[:our_user_id]
event_name = params[:name]
object_id = params[:object_id]

api = Freshbooks::API.new User.find(our_user_id)

if event_name == "callback.verify"
verifier = params[:verifier]
api.verify_callback our_user_id, verifier, object_id

elsif event_name == "invoice.create"
freshbooks_user_id = params[:user_id]
InvoiceUpdatedJob.new.delay.perform our_user_id, object_id,
freshbooks_user_id
end

respond_to do |format|
format.html { render :nothing => true}
format.json { head :no_content}
end
end

net/http rfuzz httparty
open-uri
mechanize
excon
simplehttp
em-http-request
rest-client

activeresource right_http_connection

rufus-verbs
wrest faraday

curb typhoeus
patron

httpclient eventmachine
thanks @nahi

<?xml version="1.0" encoding="utf-8"?>
<response xmlns="http://www.freshbooks.com/api/" status="ok">
<invoices page="1" per_page="10" pages="4" total="47">
<invoice>
<invoice_id>344</invoice_id>
.......
</invoice>
</invoices>
</response>

def fb_invoices
fb_collect('invoices','invoice'){|conn,page|conn.project.list(:
page=>page)}
end

def fb_collect(root_element_name, result_element_name, &block)
rv = []
conn = fb_connection()
page = 1
pages = 1
while page <= pages
temp = yield conn, page
page += 1
if !temp[root_element_name].nil? && !temp[root_element_name]
[result_element_name].nil?
if temp[root_element_name][result_element_name].kind_of?(Array)
temp[root_element_name][result_element_name].each do |elem|
rv.push(elem)
end
else
#if there's only one result, the freshbooks api returns a bare hash instead of a
hash inside an array
elem = temp[root_element_name][result_element_name]
rv.push(elem)
end
end
end
return rv
end

Tips and tricks for building api heavy ruby on rails applications

More Related Content

What's hot (19)

Viewers also liked (20)

Similar to Tips and tricks for building api heavy ruby on rails applications (20)

Tips and tricks for building api heavy ruby on rails applications