4 SAS hard drives<\/li>\n<\/ul>\nToy Photos:<\/h3>\n
![\"\"](\"https:\/\/i0.wp.com\/www.geekosas.com\/wp-content\/uploads\/2016\/11\/image_3140-300x200.jpg?resize=300%2C200&ssl=1\")
![\"\"](\"https:\/\/i0.wp.com\/www.geekosas.com\/wp-content\/uploads\/2016\/11\/hp-proliant-dl360-g6-1-300x79.jpg?resize=467%2C123\")
<\/p>\n
<\/p>\n
The Test<\/h2>\n
I will use the plyr<\/strong>, data.table<\/strong>, and reshape2<\/strong> libraries for aggregations, and doSNOW<\/strong> for parallelization. The dataset will be a table of about 30 million records similar to a mobile operator’s database (as far as I remember, these records are called CDRs stored in the CRCE of the rating platform).<\/p>\nBasically, the table will have a user identifier, a traffic type identifier, and the traffic amount.<\/p>\n
Generate the Data<\/h3>\n
We will generate 30 million CDR records in a table that will have:<\/p>\n
\n- User identifier<\/li>\n
- Traffic type<\/li>\n
- Traffic amount<\/li>\n<\/ul>\n
options(stringsAsFactors = FALSE)\nlibrary(reshape2)\nlibrary(plyr)\n\n# Parameters\nset.seed(1986) # seed set to the birth date of my dear girlfriend\ntipo_trafico = c("DATA","SMS","VOICE")\nprob_tipo_trafico = c(0.6,0.1,0.3)\nuserids = 5e4 # simulate n users\nregistros = 3e7 # number of CDR records\n\n# Random traffic amount values\ndatos_tipo_trafico =\n data.frame(\n DATA = round(runif(registros,1,1024^2)), # data session in bytes, between 1 byte and 1 MB\n SMS = rep(1,registros), # each SMS generates one record\n VOICE = round(rexp(registros,1\/120)) # call length in seconds\n )\n\n# Generate CDR records\ncdr = data.frame(\n id_cdr = 1:registros,\n userid = floor(runif(registros,1,userids+1)),\n tipo_trafico = sample(tipo_trafico,registros,replace = T,prob = prob_tipo_trafico)\n)\n\nTI = proc.time()\ncdr$trafico = datos_tipo_trafico$DATA*(cdr$tipo_trafico=="DATA") + \n datos_tipo_trafico$SMS*(cdr$tipo_trafico=="SMS") + \n datos_tipo_trafico$VOICE*(cdr$tipo_trafico=="VOICE")\nprint(proc.time()-TI)\nrm(datos_tipo_trafico) # delete random data table\ngc() # clean unused memory<\/code><\/pre>\nThe result is a table with this format:<\/p>\n
\n\n\n| <\/th>\n | id_cdr<\/th>\n | userid<\/th>\n | tipo_trafico<\/th>\n | trafico<\/th>\n<\/tr>\n<\/thead>\n |
\n\n| 1<\/td>\n | 1<\/td>\n | 5579<\/td>\n | VOICE<\/td>\n | 20<\/td>\n<\/tr>\n |
\n| 2<\/td>\n | 2<\/td>\n | 28374<\/td>\n | VOICE<\/td>\n | 82<\/td>\n<\/tr>\n |
\n| 3<\/td>\n | 3<\/td>\n | 28526<\/td>\n | VOICE<\/td>\n | 36<\/td>\n<\/tr>\n |
\n| 4<\/td>\n | 4<\/td>\n | 39179<\/td>\n | VOICE<\/td>\n | 56<\/td>\n<\/tr>\n |
\n| 5<\/td>\n | 5<\/td>\n | 14244<\/td>\n | DATA<\/td>\n | 629075<\/td>\n<\/tr>\n |
\n| 6<\/td>\n | 6<\/td>\n | 36779<\/td>\n | DATA<\/td>\n | 690397<\/td>\n<\/tr>\n |
\n| 7<\/td>\n | 7<\/td>\n | 42774<\/td>\n | DATA<\/td>\n | 175632<\/td>\n<\/tr>\n |
\n| 8<\/td>\n | 8<\/td>\n | 4276<\/td>\n | VOICE<\/td>\n | 115<\/td>\n<\/tr>\n |
\n| 9<\/td>\n | 9<\/td>\n | 4445<\/td>\n | VOICE<\/td>\n | 44<\/td>\n<\/tr>\n |
\n| 10<\/td>\n | 10<\/td>\n | 29458<\/td>\n | DATA<\/td>\n | 946171<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\nBenchmark<\/h3>\nWe will create a table of aggregated traffic per user. This table will be generated with: reshape2<\/strong>, data.table<\/strong>, and plyr<\/strong> (single\u2011thread and multi\u2011thread).<\/p>\nSome comments before showing the results:<\/p>\n \n- reshape2<\/strong> allows aggregation but offers far fewer options than the other libraries.<\/li>\n
- data.table<\/strong> is an extension of data.frame that allows the use of indices.<\/li>\n
- plyr<\/strong> can run either single\u2011thread or in parallel.<\/li>\n<\/ul>\n
The code used:<\/p>\n # using reshape2\nTI = proc.time()\ntmp = reshape2::dcast(cdr, userid ~ tipo_trafico, fun.aggregate = sum, value.var = "trafico")\nprint(proc.time()-TI)\n\n# using plyr 1 thread\nTI = proc.time()\ntmp = ddply(cdr, "userid", function(x) data.frame(DATA = sum(ifelse(x$tipo_trafico == "DATA", x$trafico, 0)),\n SMS = sum(ifelse(x$tipo_trafico == "SMS", x$trafico, 0)),\n VOICE = sum(ifelse(x$tipo_trafico == "VOICE", x$trafico, 0))\n ), .progress = T)\nprint(proc.time()-TI)\n\n# using plyr 8 threads (one per CPU)\nlibrary("doSNOW")\nnCPU = as.numeric(Sys.getenv("NUMBER_OF_PROCESSORS")[1])\ncl = makeSOCKcluster(nCPU, outfile="cl.txt")\nregisterDoSNOW(cl)\nTI = proc.time()\ntmp = ddply(cdr, "userid", function(x) data.frame(DATA = sum(ifelse(x$tipo_trafico == "DATA", x$trafico, 0)),\n SMS = sum(ifelse(x$tipo_trafico == "SMS", x$trafico, 0)),\n VOICE = sum(ifelse(x$tipo_trafico == "VOICE", x$trafico, 0))\n ), .parallel = T)\nprint(proc.time()-TI)\nstopCluster(cl)\n\n# using plyr 2 threads\nlibrary("doSNOW")\nnCPU = as.numeric(Sys.getenv("NUMBER_OF_PROCESSORS")[1])\ncl = makeSOCKcluster(2, outfile="cl.txt")\nregisterDoSNOW(cl)\nTI = proc.time()\ntmp = ddply(cdr, "userid", function(x) data.frame(DATA = sum(ifelse(x$tipo_trafico == "DATA", x$trafico, 0)),\n SMS = sum(ifelse(x$tipo_trafico == "SMS", x$trafico, 0)),\n VOICE = sum(ifelse(x$tipo_trafico == "VOICE", x$trafico, 0))\n ), .parallel = T)\nprint(proc.time()-TI)\nstopCluster(cl)\n\n# using data.table\nlibrary(data.table)\ncdr_dt = data.table(cdr, key = "userid")\nTI = proc.time()\ntmp = cdr_dt[, list(DATA = sum(ifelse(tipo_trafico == "DATA", trafico, 0)),\n SMS = sum(ifelse(tipo_trafico == "SMS", trafico, 0)),\n VOICE = sum(ifelse(tipo_trafico == "VOICE", trafico, 0))\n ), by = "userid"]\nprint(proc.time()-TI)<\/code><\/pre>\nResults<\/h2>\n\n\n\n| library<\/th>\n | function<\/th>\n | threads<\/th>\n | seconds<\/th>\n<\/tr>\n<\/thead>\n | \n\n| plyr<\/td>\n | ddply<\/td>\n | 1<\/td>\n | 9.18<\/td>\n<\/tr>\n | \n| plyr<\/td>\n | ddply<\/td>\n | 2<\/td>\n | 78.37<\/td>\n<\/tr>\n | \n| plyr<\/td>\n | ddply<\/td>\n | 8<\/td>\n | 146.04<\/td>\n<\/tr>\n | \n| reshape2<\/td>\n | dcast<\/td>\n | 1<\/td>\n | 17.31<\/td>\n<\/tr>\n | \n| data.table<\/td>\n | <\/td>\n | 1<\/td>\n | 12.94<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\nConclusions<\/h2>\nreshape2<\/strong>, despite being a library specialized in this type of transformation, turned out to be slower than plyr<\/strong>.<\/p>\nplyr<\/strong> lost performance as more threads were added. This is because for simple operations, the cost of parallelization is higher than not using the other CPUs.<\/p>\ndata.table<\/strong>, despite using indices, turned out to be slower than plyr<\/strong>. When complex algorithms are used for simple situations, performance can sometimes be lost.<\/p>\nFor the Future…<\/h2>\nIn the future, I will run these tests using larger datasets and different numbers of users to see what results are obtained.<\/p>\n","protected":false},"excerpt":{"rendered":" Introduction A few days ago, while walking around, I saw that they were selling a used HP Proliant DL360 G6. For those who don’t know, […]<\/a><\/p>\n<\/div>","protected":false},"author":1,"featured_media":2137,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_jetpack_newsletter_access":"","_jetpack_dont_email_post_to_subs":false,"_jetpack_newsletter_tier_id":0,"_jetpack_memberships_contains_paywalled_content":false,"_jetpack_memberships_contains_paid_content":false,"footnotes":"","jetpack_publicize_message":"","jetpack_publicize_feature_enabled":true,"jetpack_social_post_already_shared":true,"jetpack_social_options":{"image_generator_settings":{"template":"highway","default_image_id":0,"font":"","enabled":false},"version":2},"jetpack_post_was_ever_published":false},"categories":[1],"tags":[],"class_list":["post-3281","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-sin-categoria"],"jetpack_publicize_connections":[],"jetpack_featured_media_url":"https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2016\/11\/RStudio-Ball.png?fit=1000%2C1000&ssl=1","jetpack_sharing_enabled":true,"jetpack_shortlink":"https:\/\/wp.me\/p8vjqF-QV","jetpack-related-posts":[{"id":3262,"url":"https:\/\/geekosas.com\/index.php\/2016\/05\/23\/arduino-2\/","url_meta":{"origin":3281,"position":0},"title":"Arduino","author":"Daniel Fischer","date":"2016-05-23","format":false,"excerpt":"It's hard to start this article. I've been playing with my new Arduino Starter Kit for a few days, and I think the best way to define it is: an electronic Lego. An excellent toy for those who enjoyed the blocks years ago. Arduino is a microcontroller easily programmable from\u2026","rel":"","context":"In "Sin categor\u00eda"","block_context":{"text":"Sin categor\u00eda","link":"https:\/\/geekosas.com\/index.php\/category\/sin-categoria\/"},"img":{"alt_text":"","src":"https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2016\/04\/ardr3a.jpg?fit=1200%2C829&ssl=1&resize=350%2C200","width":350,"height":200,"srcset":"https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2016\/04\/ardr3a.jpg?fit=1200%2C829&ssl=1&resize=350%2C200 1x, https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2016\/04\/ardr3a.jpg?fit=1200%2C829&ssl=1&resize=525%2C300 1.5x, https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2016\/04\/ardr3a.jpg?fit=1200%2C829&ssl=1&resize=700%2C400 2x, https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2016\/04\/ardr3a.jpg?fit=1200%2C829&ssl=1&resize=1050%2C600 3x"},"classes":[]},{"id":3264,"url":"https:\/\/geekosas.com\/index.php\/2016\/05\/23\/how-to-choose-a-cell-phone\/","url_meta":{"origin":3281,"position":1},"title":"How to choose a cell phone?","author":"Daniel Fischer","date":"2016-05-23","format":false,"excerpt":"We spend more time with our cell phone than with our family. In fact, we use it to communicate with them, and for many it is an essential work tool. A cell phone, whether you want to admit it or not, is a life companion that, depending on the relationship,\u2026","rel":"","context":"In "Sin categor\u00eda"","block_context":{"text":"Sin categor\u00eda","link":"https:\/\/geekosas.com\/index.php\/category\/sin-categoria\/"},"img":{"alt_text":"","src":"https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2016\/04\/mensagens_amor_namorados_para_celular.jpg?fit=400%2C225&ssl=1&resize=350%2C200","width":350,"height":200},"classes":[]},{"id":3223,"url":"https:\/\/geekosas.com\/index.php\/2020\/05\/14\/insert-records-into-the-database-at-full-speed\/","url_meta":{"origin":3281,"position":2},"title":"Insert Records into the Database at Full Speed","author":"Daniel Fischer","date":"2020-05-14","format":false,"excerpt":"ETL tools are very useful for performing automated and recurring data transformation processes; they are characterized by performing three tasks: (E) Extract: Connect to one or more sources and extract data. (T) Transform: Transform or manipulate the data. (L) Load: Load the transformed data into the final repository. That is\u2026","rel":"","context":"In "Sin categor\u00eda"","block_context":{"text":"Sin categor\u00eda","link":"https:\/\/geekosas.com\/index.php\/category\/sin-categoria\/"},"img":{"alt_text":"","src":"https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2021\/09\/patch.png?fit=728%2C380&ssl=1&resize=350%2C200","width":350,"height":200,"srcset":"https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2021\/09\/patch.png?fit=728%2C380&ssl=1&resize=350%2C200 1x, https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2021\/09\/patch.png?fit=728%2C380&ssl=1&resize=525%2C300 1.5x, https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2021\/09\/patch.png?fit=728%2C380&ssl=1&resize=700%2C400 2x"},"classes":[]},{"id":3285,"url":"https:\/\/geekosas.com\/index.php\/2017\/05\/23\/video-games-and-statistics\/","url_meta":{"origin":3281,"position":3},"title":"Video Games and Statistics","author":"Daniel Fischer","date":"2017-05-23","format":false,"excerpt":"The video game industry has grown exponentially. New game genres, new business models, new types of gamers, and new devices for playing have been created, but what hasn't changed is that so\u2011called triple\u2011A games still exist. Triple\u2011A games are characterized by belonging to companies with a large budget to invest\u2026","rel":"","context":"In "Sin categor\u00eda"","block_context":{"text":"Sin categor\u00eda","link":"https:\/\/geekosas.com\/index.php\/category\/sin-categoria\/"},"img":{"alt_text":"","src":"https:\/\/i0.wp.com\/www.geekosas.com\/wp-content\/uploads\/2017\/02\/fifa-300x121.jpg?resize=350%2C200","width":350,"height":200,"srcset":"https:\/\/i0.wp.com\/www.geekosas.com\/wp-content\/uploads\/2017\/02\/fifa-300x121.jpg?resize=350%2C200 1x, https:\/\/i0.wp.com\/www.geekosas.com\/wp-content\/uploads\/2017\/02\/fifa-300x121.jpg?resize=525%2C300 1.5x, https:\/\/i0.wp.com\/www.geekosas.com\/wp-content\/uploads\/2017\/02\/fifa-300x121.jpg?resize=700%2C400 2x"},"classes":[]},{"id":3269,"url":"https:\/\/geekosas.com\/index.php\/2016\/05\/23\/playing-with-neural-networks\/","url_meta":{"origin":3281,"position":4},"title":"Playing with Neural Networks","author":"Daniel Fischer","date":"2016-05-23","format":false,"excerpt":"A few days ago I wrote about neural networks, trying to explain in a simple way how they work and why this type of technology is called \"artificial intelligence.\" The important thing about this technique and what makes it so revolutionary is the concept of training, but it is often\u2026","rel":"","context":"In "Sin categor\u00eda"","block_context":{"text":"Sin categor\u00eda","link":"https:\/\/geekosas.com\/index.php\/category\/sin-categoria\/"},"img":{"alt_text":"","src":"https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2016\/04\/tensorflow.png?fit=1200%2C715&ssl=1&resize=350%2C200","width":350,"height":200,"srcset":"https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2016\/04\/tensorflow.png?fit=1200%2C715&ssl=1&resize=350%2C200 1x, https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2016\/04\/tensorflow.png?fit=1200%2C715&ssl=1&resize=525%2C300 1.5x, https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2016\/04\/tensorflow.png?fit=1200%2C715&ssl=1&resize=700%2C400 2x, https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2016\/04\/tensorflow.png?fit=1200%2C715&ssl=1&resize=1050%2C600 3x"},"classes":[]},{"id":3227,"url":"https:\/\/geekosas.com\/index.php\/2026\/05\/13\/como-se-repartieron-los-votos-en-la-segunda-vuelta-en-chile\/","url_meta":{"origin":3281,"position":5},"title":"How were the votes distributed in the runoff election in Chile?","author":"Daniel Fischer","date":"2026-05-13","format":false,"excerpt":"Acknowledgments: I would like to begin this article by thanking Hernan Aburto (huaburto@gmail.com) for his assistance. We worked together to refine the model and polish the conclusions. His version of the article on his blog: https:\/\/www.grialprofano.cl\/flyto\/97\/que-paso-con-los-votos-de-la-primera-vuelta-en-el-balotaje\/ How were the votes distributed? In the most recent elections held on Sunday, December\u2026","rel":"","context":"In "Sin categor\u00eda"","block_context":{"text":"Sin categor\u00eda","link":"https:\/\/geekosas.com\/index.php\/category\/sin-categoria\/"},"img":{"alt_text":"","src":"https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2021\/12\/FRCN3F5FY6BYLWSLIY4NSVWH5E.jpg?fit=900%2C600&ssl=1&resize=350%2C200","width":350,"height":200,"srcset":"https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2021\/12\/FRCN3F5FY6BYLWSLIY4NSVWH5E.jpg?fit=900%2C600&ssl=1&resize=350%2C200 1x, https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2021\/12\/FRCN3F5FY6BYLWSLIY4NSVWH5E.jpg?fit=900%2C600&ssl=1&resize=525%2C300 1.5x, https:\/\/i0.wp.com\/geekosas.com\/wp-content\/uploads\/2021\/12\/FRCN3F5FY6BYLWSLIY4NSVWH5E.jpg?fit=900%2C600&ssl=1&resize=700%2C400 2x"},"classes":[]}],"_links":{"self":[{"href":"https:\/\/geekosas.com\/index.php\/wp-json\/wp\/v2\/posts\/3281","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/geekosas.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/geekosas.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/geekosas.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/geekosas.com\/index.php\/wp-json\/wp\/v2\/comments?post=3281"}],"version-history":[{"count":1,"href":"https:\/\/geekosas.com\/index.php\/wp-json\/wp\/v2\/posts\/3281\/revisions"}],"predecessor-version":[{"id":3282,"href":"https:\/\/geekosas.com\/index.php\/wp-json\/wp\/v2\/posts\/3281\/revisions\/3282"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/geekosas.com\/index.php\/wp-json\/wp\/v2\/media\/2137"}],"wp:attachment":[{"href":"https:\/\/geekosas.com\/index.php\/wp-json\/wp\/v2\/media?parent=3281"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/geekosas.com\/index.php\/wp-json\/wp\/v2\/categories?post=3281"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/geekosas.com\/index.php\/wp-json\/wp\/v2\/tags?post=3281"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}} |
|