R meets Hadoop
Download as KEY, PDF5 likes5,351 views
- The document describes a MapReduce workflow for analyzing airline flight data from multiple text files. - The map function parses the raw data by date, carrier, origin, destination, and converts time fields to datetime objects. - The reduce function aggregates the data by origin and destination airports to calculate inbound, outbound, and total flights. - The results are written to a new folder and then read back into R for further analysis and ranking of airports by flight volumes.
![map <- expression({
library(openNLP)
f <- table(tokenize(unlist(map.values), language = "en"))
n <- names(f)
p <- as.numeric(f)
sapply(seq_along(n),function(r) rhcollect(n[r],p[r]))
})
reduce <- expression(
pre = { total <- 0},
reduce = { total <- total+sum(unlist(reduce.values)) },
post = { rhcollect(reduce.key,total) }
)
z <- rhmr(map=map, reduce=reduce, inout=c("text","sequence")
,ifolder=”/tmp/input”, ofolder=”/tmp/output”)
rhex(z)
17](https://image.slidesharecdn.com/rhipe-110530074154-phpapp02/85/R-meets-Hadoop-17-320.jpg)
![> results <- rhread("/tmp/output")
> results <- data.frame(word=unlist(lapply(results,"[[",1))’
+ ,count =unlist (lapply(results,"[[",2)))
> results <- (results[order(results$count, decreasing=TRUE), ])
> head(results)
word count
13 . 2080
439 the 1101
11 , 760
32 a 701
153 to 658
28 I 651
> results[results["word"] == "FACEBOOK", ]
word count
3221 FACEBOOK 6
> results[results["word"] == "Facebook", ]
word count
3223 Facebook 39
> results[results["word"] == "facebook", ]
word count
3389 facebook 6
18](https://image.slidesharecdn.com/rhipe-110530074154-phpapp02/85/R-meets-Hadoop-18-320.jpg)
![setup <- expression({
convertHHMM <- function(s){
t(sapply(s,function(r){
l=nchar(r)
if(l==4) c(substr(r,1,2),substr(r,3,4))
else if(l==3) c(substr(r,1,1),substr(r,2,3))
else c('0','0')
})
)}
})
map <- expression({
y <- do.call("rbind",lapply(map.values,function(r){
if(substr(r,1,4)!='Year') strsplit(r,",")[[1]]
}))
mu <- rep(1,nrow(y));yr <- y[,1]; mn=y[,2];dy=y[,3]
hr <- convertHHMM(y[,5])
depart <- ISOdatetime(year=yr,month=mn,day=dy,hour=hr[,1],min=hr[,2],sec=mu)
hr <- convertHHMM(y[,6])
sdepart <- ISOdatetime(year=yr,month=mn,day=dy,hour=hr[,1],min=hr[,2],sec=mu)
hr <- convertHHMM(y[,7])
arrive <- ISOdatetime(year=yr,month=mn,day=dy,hour=hr[,1],min=hr[,2],sec=mu)
hr <- convertHHMM(y[,8])
sarrive <- ISOdatetime(year=yr,month=mn,day=dy,hour=hr[,1],min=hr[,2],sec=mu)
d <- data.frame(depart= depart,sdepart = sdepart, arrive = arrive,sarrive =sarrive
,carrier = y[,9],origin = y[,17], dest=y[,18],dist = y[,19], year=yr, month=mn, day=dy
,cancelled=y[,22], stringsAsFactors=FALSE)
d <- d[order(d$sdepart),]
rhcollect(d[c(1,nrow(d)),"sdepart"],d)
})
reduce <- expression(
reduce = { lapply(reduce.values, function(i) rhcollect(reduce.key,i))}
)
z <- rhmr(map=map,reduce=reduce,setup=setup,inout=c("text","sequence")
,ifolder="/airline/data/",ofolder="/airline/blocks",mapred=mapred,orderby="numeric") 21
rhex(z)](https://image.slidesharecdn.com/rhipe-110530074154-phpapp02/85/R-meets-Hadoop-21-320.jpg)
![map <- expression({
a <- do.call("rbind",map.values)
inbound <- table(a[,'origin'])
outbound <- table(a[,'dest'])
total <- table(unlist(c(a[,'origin'],a['dest'])))
for (n in names(total)) {
inb <- if(is.na(inbound[n])) 0 else inbound[n]
ob <- if(is.na(outbound[n])) 0 else outbound[n]
rhcollect(n, c(inb,ob, total[n]))
}
})
reduce <- expression(
pre = { sums <- c(0,0,0) },
reduce = { sums <- sums+apply(do.call("rbind",reduce.values),2,sum) },
post = { rhcollect(reduce.key, sums) }
)
z <- rhmr(map=map,reduce=reduce,combiner=TRUE,inout=c("sequence","sequence")
,ifolder="/airline/blocks/",ofolder="/airline/volume")
rhex(z,async=TRUE)
22](https://image.slidesharecdn.com/rhipe-110530074154-phpapp02/85/R-meets-Hadoop-22-320.jpg)
![> counts <- rhread("/airline/volume")
> aircode <- unlist(lapply(counts, "[[",1))
> count <- do.call("rbind",lapply(counts,"[[",2))
> results <- data.frame(aircode=aircode,
+ inb=count[,1],oub=count[,2],all=count[,3]
+ ,stringsAsFactors=FALSE)
> results <- results[order(results$all,decreasing=TRUE),]
> ap <- read.table("~/tmp/airports.csv",sep=",",header=TRUE,
+ stringsAsFactors=FALSE,na.strings="XYZB")
> results$airport <- sapply(results$aircode,function(r){
+ nam <- ap[ap$iata==r,'airport']
+ if(length(nam)==0) r else nam
+ })
> results[1:10,]
aircode inb oub all airport
243 ORD 6597442 6638035 13235477 Chicago O'Hare International
21 ATL 6100953 6094186 12195139 William B Hartsfield-Atlanta Intl
91 DFW 5710980 5745593 11456573 Dallas-Fort Worth International
182 LAX 4089012 4086930 8175942 Los Angeles International
254 PHX 3491077 3497764 6988841 Phoenix Sky Harbor International
89 DEN 3319905 3335222 6655127 Denver Intl
97 DTW 2979158 2997138 5976296 Detroit Metropolitan-Wayne County
156 IAH 2884518 2889971 5774489 George Bush Intercontinental
230 MSP 2754997 2765191 5520188 Minneapolis-St Paul Intl
300 SFO 2733910 2725676 5459586 San Francisco International
23](https://image.slidesharecdn.com/rhipe-110530074154-phpapp02/85/R-meets-Hadoop-23-320.jpg)
![map <- expression({
a <- do.call("rbind",map.values)
y <- table(apply(a[,c("origin","dest")],1,function(r){
paste(sort(r),collapse=",")
}))
for(i in 1:length(y)){
p <- strsplit(names(y)[[i]],",")[[1]]
rhcollect(p,y[[1]])
}
})
reduce <- expression(
pre = {sums <- 0},
reduce = {sums <- sums+sum(unlist(reduce.values))},
post = { rhcollect(reduce.key, sums) }
)
z <- rhmr(map=map,reduce=reduce,combiner=TRUE,inout=c("sequence","sequence")
,ifolder="/airline/blocks/",ofolder="/airline/ijjoin")
z=rhex(z)
25](https://image.slidesharecdn.com/rhipe-110530074154-phpapp02/85/R-meets-Hadoop-25-320.jpg)
![> b=rhread("/airline/ijjoin")
> y <- do.call("rbind",lapply(b,"[[",1))
> results <- data.frame(a=y[,1],b=y[,2],count=
+ do.call("rbind",lapply(b,"[[",2)),stringsAsFactors=FALSE)
> results <- results[order(results$count,decreasing=TRUE),]
> results$cumprop <- cumsum(results$count)/sum(results$count)
> a.lat <- t(sapply(results$a,function(r){
+ ap[ap$iata==r,c('lat','long')]
+ }))
> results$a.lat <- unlist(a.lat[,'lat'])
> results$a.long <- unlist(a.lat[,'long'])
> b.lat <- t(sapply(results$b,function(r){
+ ap[ap$iata==r,c('lat','long')]
+ }))
> b.lat["CBM",] <- c(0,0)
> results$b.lat <- unlist(b.lat[,'lat'])
> results$b.long <- unlist(b.lat[,'long'])
> head(results)
a b count cumprop a.lat a.long b.lat b.long
418 ATL ORD 141465 0.001546158 33.64044 -84.42694 41.97960 -87.90446
2079 DEN DFW 138892 0.003064195 39.85841 -104.66700 32.89595 -97.03720
331 ATL DFW 135357 0.004543595 33.64044 -84.42694 32.89595 -97.03720
2221 DFW IAH 134508 0.006013716 32.89595 -97.03720 29.98047 -95.33972
3568 LAS LAX 132333 0.007460065 36.08036 -115.15233 33.94254 -118.40807
2409 DTW ORD 130065 0.008881626 42.21206 -83.34884 41.97960 -87.90446
26](https://image.slidesharecdn.com/rhipe-110530074154-phpapp02/85/R-meets-Hadoop-26-320.jpg)
![Some R Examples[R table and Graphics] -Advanced Data Visualization in R (Some...](https://cdn.slidesharecdn.com/ss_thumbnails/exampless-160922204223-thumbnail.jpg?width=560&fit=bounds)
More Related Content
What's hot (20)
Similar to R meets Hadoop (20)
More from Hidekazu Tanaka (10)
Recently uploaded (20)
R meets Hadoop
- 6. 6
- 7. 7
- 8. 8
- 9. Blocks (Input Data) Parallel Partition Node 1 Node 2 Node 3 (MAP) Network Transfer Parallel Recombine Node 1 Node 2 Node 3 (REDUCE) Output Data 9
- 10. 10
- 11. 11
- 12. 12
- 13. 13
- 14. 14
- 15. • R CMD INSTALL Rhipe_version.tar.gz 15
- 16. map <- expression({ # }) reduce <- expression( pre = {}, reduce = {}, post = {} ) z <- rhmr(map=map, reduce=reduce, inout=c("text","sequence") ,ifolder=”/tmp/input”, ofolder=”/tmp/output”) rhex(z) results <- rhread(“/tmp/output”) 16
- 17. map <- expression({ library(openNLP) f <- table(tokenize(unlist(map.values), language = "en")) n <- names(f) p <- as.numeric(f) sapply(seq_along(n),function(r) rhcollect(n[r],p[r])) }) reduce <- expression( pre = { total <- 0}, reduce = { total <- total+sum(unlist(reduce.values)) }, post = { rhcollect(reduce.key,total) } ) z <- rhmr(map=map, reduce=reduce, inout=c("text","sequence") ,ifolder=”/tmp/input”, ofolder=”/tmp/output”) rhex(z) 17
- 18. > results <- rhread("/tmp/output") > results <- data.frame(word=unlist(lapply(results,"[[",1))’ + ,count =unlist (lapply(results,"[[",2))) > results <- (results[order(results$count, decreasing=TRUE), ]) > head(results) word count 13 . 2080 439 the 1101 11 , 760 32 a 701 153 to 658 28 I 651 > results[results["word"] == "FACEBOOK", ] word count 3221 FACEBOOK 6 > results[results["word"] == "Facebook", ] word count 3223 Facebook 39 > results[results["word"] == "facebook", ] word count 3389 facebook 6 18
- 19. 19
- 20. map <- expression({ msys <- function(on){ system(sprintf("wget %s --directory-prefix ./tmp 2> ./errors",on)) if(length(grep("(failed)|(unable)",readLines("./errors")))>0){ stop(paste(readLines("./errors"),collapse="n")) }} lapply(map.values,function(x){ x=1986+x on <- sprintf("http://stat-computing.org/dataexpo/2009/%s.csv.bz2",x) fn <- sprintf("./tmp/%s.csv.bz2",x) rhstatus(sprintf("Downloading %s", on)) msys(on) rhstatus(sprintf("Downloaded %s", on)) system(sprintf('bunzip2 %s',fn)) rhstatus(sprintf("Unzipped %s", on)) rhcounter("FILES",x,1) rhcounter("FILES","_ALL_",1) }) }) z <- rhmr(map=map,ofolder="/airline/data",inout=c("lapply"), N=length(1987:2008), mapred=list(mapred.reduce.tasks=0,mapred.task.timeout=0),copyFiles=TRUE) j <- rhex(z,async=TRUE) 20
- 21. setup <- expression({ convertHHMM <- function(s){ t(sapply(s,function(r){ l=nchar(r) if(l==4) c(substr(r,1,2),substr(r,3,4)) else if(l==3) c(substr(r,1,1),substr(r,2,3)) else c('0','0') }) )} }) map <- expression({ y <- do.call("rbind",lapply(map.values,function(r){ if(substr(r,1,4)!='Year') strsplit(r,",")[[1]] })) mu <- rep(1,nrow(y));yr <- y[,1]; mn=y[,2];dy=y[,3] hr <- convertHHMM(y[,5]) depart <- ISOdatetime(year=yr,month=mn,day=dy,hour=hr[,1],min=hr[,2],sec=mu) hr <- convertHHMM(y[,6]) sdepart <- ISOdatetime(year=yr,month=mn,day=dy,hour=hr[,1],min=hr[,2],sec=mu) hr <- convertHHMM(y[,7]) arrive <- ISOdatetime(year=yr,month=mn,day=dy,hour=hr[,1],min=hr[,2],sec=mu) hr <- convertHHMM(y[,8]) sarrive <- ISOdatetime(year=yr,month=mn,day=dy,hour=hr[,1],min=hr[,2],sec=mu) d <- data.frame(depart= depart,sdepart = sdepart, arrive = arrive,sarrive =sarrive ,carrier = y[,9],origin = y[,17], dest=y[,18],dist = y[,19], year=yr, month=mn, day=dy ,cancelled=y[,22], stringsAsFactors=FALSE) d <- d[order(d$sdepart),] rhcollect(d[c(1,nrow(d)),"sdepart"],d) }) reduce <- expression( reduce = { lapply(reduce.values, function(i) rhcollect(reduce.key,i))} ) z <- rhmr(map=map,reduce=reduce,setup=setup,inout=c("text","sequence") ,ifolder="/airline/data/",ofolder="/airline/blocks",mapred=mapred,orderby="numeric") 21 rhex(z)
- 22. map <- expression({ a <- do.call("rbind",map.values) inbound <- table(a[,'origin']) outbound <- table(a[,'dest']) total <- table(unlist(c(a[,'origin'],a['dest']))) for (n in names(total)) { inb <- if(is.na(inbound[n])) 0 else inbound[n] ob <- if(is.na(outbound[n])) 0 else outbound[n] rhcollect(n, c(inb,ob, total[n])) } }) reduce <- expression( pre = { sums <- c(0,0,0) }, reduce = { sums <- sums+apply(do.call("rbind",reduce.values),2,sum) }, post = { rhcollect(reduce.key, sums) } ) z <- rhmr(map=map,reduce=reduce,combiner=TRUE,inout=c("sequence","sequence") ,ifolder="/airline/blocks/",ofolder="/airline/volume") rhex(z,async=TRUE) 22
- 23. > counts <- rhread("/airline/volume") > aircode <- unlist(lapply(counts, "[[",1)) > count <- do.call("rbind",lapply(counts,"[[",2)) > results <- data.frame(aircode=aircode, + inb=count[,1],oub=count[,2],all=count[,3] + ,stringsAsFactors=FALSE) > results <- results[order(results$all,decreasing=TRUE),] > ap <- read.table("~/tmp/airports.csv",sep=",",header=TRUE, + stringsAsFactors=FALSE,na.strings="XYZB") > results$airport <- sapply(results$aircode,function(r){ + nam <- ap[ap$iata==r,'airport'] + if(length(nam)==0) r else nam + }) > results[1:10,] aircode inb oub all airport 243 ORD 6597442 6638035 13235477 Chicago O'Hare International 21 ATL 6100953 6094186 12195139 William B Hartsfield-Atlanta Intl 91 DFW 5710980 5745593 11456573 Dallas-Fort Worth International 182 LAX 4089012 4086930 8175942 Los Angeles International 254 PHX 3491077 3497764 6988841 Phoenix Sky Harbor International 89 DEN 3319905 3335222 6655127 Denver Intl 97 DTW 2979158 2997138 5976296 Detroit Metropolitan-Wayne County 156 IAH 2884518 2889971 5774489 George Bush Intercontinental 230 MSP 2754997 2765191 5520188 Minneapolis-St Paul Intl 300 SFO 2733910 2725676 5459586 San Francisco International 23
- 25. map <- expression({ a <- do.call("rbind",map.values) y <- table(apply(a[,c("origin","dest")],1,function(r){ paste(sort(r),collapse=",") })) for(i in 1:length(y)){ p <- strsplit(names(y)[[i]],",")[[1]] rhcollect(p,y[[1]]) } }) reduce <- expression( pre = {sums <- 0}, reduce = {sums <- sums+sum(unlist(reduce.values))}, post = { rhcollect(reduce.key, sums) } ) z <- rhmr(map=map,reduce=reduce,combiner=TRUE,inout=c("sequence","sequence") ,ifolder="/airline/blocks/",ofolder="/airline/ijjoin") z=rhex(z) 25
- 26. > b=rhread("/airline/ijjoin") > y <- do.call("rbind",lapply(b,"[[",1)) > results <- data.frame(a=y[,1],b=y[,2],count= + do.call("rbind",lapply(b,"[[",2)),stringsAsFactors=FALSE) > results <- results[order(results$count,decreasing=TRUE),] > results$cumprop <- cumsum(results$count)/sum(results$count) > a.lat <- t(sapply(results$a,function(r){ + ap[ap$iata==r,c('lat','long')] + })) > results$a.lat <- unlist(a.lat[,'lat']) > results$a.long <- unlist(a.lat[,'long']) > b.lat <- t(sapply(results$b,function(r){ + ap[ap$iata==r,c('lat','long')] + })) > b.lat["CBM",] <- c(0,0) > results$b.lat <- unlist(b.lat[,'lat']) > results$b.long <- unlist(b.lat[,'long']) > head(results) a b count cumprop a.lat a.long b.lat b.long 418 ATL ORD 141465 0.001546158 33.64044 -84.42694 41.97960 -87.90446 2079 DEN DFW 138892 0.003064195 39.85841 -104.66700 32.89595 -97.03720 331 ATL DFW 135357 0.004543595 33.64044 -84.42694 32.89595 -97.03720 2221 DFW IAH 134508 0.006013716 32.89595 -97.03720 29.98047 -95.33972 3568 LAS LAX 132333 0.007460065 36.08036 -115.15233 33.94254 -118.40807 2409 DTW ORD 130065 0.008881626 42.21206 -83.34884 41.97960 -87.90446 26
- 27. 27
- 28. 28