Async or Sync Log in Erlang - Limit the Load of Singleton Process

In a previous blog: A Case Study of Scalability Related "Out of memory" Crash in Erlang, I described a scalability related issue, which was coming from a singleton async logger process. A singleton Erlang process can not benefit from multiple-core scalability.

I then did some testing on disk_log, which fortunately has sync log functions: log/2 and blog/2. Whenever a process calls disk_log:blog/2, the requesting process will monitor the logger process until it returns a result or failure. The piece of code is like:

monitor_request(Pid, Req) ->
    Ref = erlang:monitor(process, Pid),
    Pid ! {self(), Req},
    receive 
	{'DOWN', Ref, process, Pid, _Info} ->
	    {error, no_such_log};
	{disk_log, Pid, Reply} ->
	    erlang:demonitor(Ref),
	    receive 
		{'DOWN', Ref, process, Pid, _Reason} ->
		    Reply
	    after 0 ->
                    Reply
	    end
    end.

Where Pid is the logger's process id.

This piece of code shows how to interactive synchronously between processes.

Under sync mode, there may be a lot of simultaneous requesting processes request the logger process to log message asynchronously, but each requesting process will wait the logger's work done before it requests next log, i.e. each requesting process requests the logger to log message synchronously. Upon this sync mode, we can guarantee the logger's message queue length won't exceed the number of simultaneous requesting processes.

I wrote some testing code: dlogger.erl

-module(dlogger).

-export([sync/2,
         async/2,
         proc_loop/4
        ]).

-define(LogName, blog).
-define(LogFile, "b.log").

% 100000, 10
sync(N_Msg, N_Procs) ->
   test(N_Msg, N_Procs, fun disk_log:blog/2).

async(N_Msg, N_Procs) ->
   test(N_Msg, N_Procs, fun disk_log:balog/2).


test(N_Msg, N_Procs, FunLog) ->
   MsgPerProc = round(N_Msg / N_Procs),
   Collector = init(N_Procs),
   LogPid = logger_pid(?LogName),
   io:format("logger pid: ~p~n", [LogPid]),
   Workers = [spawn(?MODULE, proc_loop, [Collector, MsgPerProc, LogPid, FunLog]) || _I <- lists:seq(1, N_Procs)],
   Start = now(),
   [Worker ! start || Worker <- Workers],
   %% don't terminate, wait here, until all tasks done.
   receive
      {sent_done, MaxMQLen, MaxMem} ->
         probe_logger(LogPid, MaxMQLen, MaxMem),
         [exit(Worker, kill) || Worker <- Workers],
         io:format("Time: ~10.2f ms~n", [timer:now_diff(now(), Start) / 1000])
   end.

init(N_Procs) ->
   disk_log:close(?LogName),
   disk_log:open([{name, ?LogName},
                  {file, ?LogFile},
                  {format, external}]),
   MainPid = self(),
   Collector = spawn(fun() -> collect(MainPid, N_Procs, 0, 0, 0, 0) end),
   Collector.

collect(MainPid, N_Procs, N_Finished, _N_Msg, MaxMQLen, MaxMem) when N_Procs == N_Finished ->
   MainPid ! {sent_done, MaxMQLen, MaxMem};
collect(MainPid, N_Procs, N_Finished, N_Msg, MaxMQLen, MaxMem) ->
   receive
      {Pid, sent_done, MQLen, _Mem} ->
         io:format("==== QLen ~p. Proc ~p finished, total finished: ~p ====~n", [MQLen, Pid, N_Finished + 1]),
         collect(MainPid, N_Procs, N_Finished + 1, N_Msg, MaxMQLen, MaxMem);
      {Pid, I, MQLen, Mem} ->
         %io:format("Processed/Qlen ~p/~p msgs. Logger mem is ~p. proc ~p: No.~p msgs sent~n", [N_Msg + 1, MQLen, Mem, Pid, I]),
         MaxMQLen1 = if MQLen > MaxMQLen -> MQLen; true -> MaxMQLen end,
         MaxMem1 = if Mem > MaxMem -> Mem; true -> MaxMem end,
         collect(MainPid, N_Procs, N_Finished, N_Msg + 1, MaxMQLen1, MaxMem1)
   end.

proc_loop(Collector, N_Msg, LogPid, LogFun) ->
   receive
      start ->
         do_proc_work(Collector, N_Msg, LogPid, LogFun, do_log)
   end.

do_proc_work(Collector, I, LogPid, LogFun, WorkType) ->
   Date = httpd_util:rfc1123_date(calendar:local_time()),
   MQLen = logger_mqlen(LogPid),
   Mem = logger_mem(LogPid),
   Msg = io_lib:format("logged in ~p, logger qlen is ~p, total mem is ~p\n",
                       [self(), MQLen, Mem]),
   Msg1 = list_to_binary([<<"=INFO REPORT==== ">>, Date, <<" ===\n">>, Msg, <<"\n">>]),

   WorkType1 = if WorkType == do_log ->
                     LogFun(?LogName, Msg1),
                     io:format("", []), % sync the io between collector if any 
                     Collector ! {self(), I, MQLen, Mem},
                     io:format("sent one msg, qlen:~p, mem:~p~n", [MQLen, Mem]),
                     if I =< 1 ->
                           Collector ! {self(), sent_done, MQLen, Mem},
                           io:format("~p sent done, qlen:~p, mem:~p~n", [self(), MQLen, Mem]),
                           keep_live;
                        true -> do_log
                     end;
                  true -> keep_live
               end,
   do_proc_work(Collector, I - 1, LogPid, LogFun, WorkType1).

probe_logger(Pid, MaxMQLen, MaxMem) ->
   MQLen = logger_mqlen(Pid),
   Mem = logger_mem(Pid),
   MaxMQLen1 = if MQLen > MaxMQLen -> MQLen; true -> MaxMQLen end,
   MaxMem1 = if Mem > MaxMem -> Mem; true -> MaxMem end,
   io:format("qlen is ~p, max qlen is ~p, max mem is ~p~n", [MQLen, MaxMQLen1, MaxMem1]),
   if MQLen == 0 -> done;
      true ->
         timer:sleep(10),
         probe_logger(Pid, MaxMQLen, MaxMem)
   end.

%% === helper ===
logger_pid(Log) ->
   case disk_log_server:get_log_pids(Log) of
      undefined ->
         undefined;
      {local, Pid} ->
         Pid;
      {distributed, [Pid|_Pids]} ->
         Pid
   end.

logger_mqlen(undefined) -> 0;
logger_mqlen(Pid) ->
   case process_info(Pid, message_queue_len) of
      {message_queue_len, Val} when is_integer(Val) -> Val;
      _ -> 0
   end.

logger_mem(undefined) -> 0;
logger_mem(Pid) ->
   case process_info(Pid, memory) of
      {memory, Val} when is_integer(Val) -> Val;
      _ -> 0
   end.

You can always use process_info/2 or process_info/1 to probe the information of a process. In above code, we will probe the logger process's message queue length and memory via logger_mqlen/1 and logger_mem/1 when necessary.

I wrote the code as it, so I can create thousands of processes first, then each process will repeatedly request to log message MsgPerProc times, and keep alive after all messages have been sent.

And, to evaluate the actual task time, when all requesting processes have finished sending log messages, a probe_logger/1 function will confirm all messages in logger queue have been processed.

Here's the result:

> dlogger:sync(1000000, 1000) % 1 million log messages under 1000 requesting processes:
qlen is 0, max qlen is 999, max mem is 690,400
Time:  861286.24 ms

> dlogger:async(1000000, 1000) % 1 million log messages under 1000 requesting processes:
qlen is 0, max qlen is 68487, max mem is 75,830,616
Time: 2156351.45 ms

The performance in async mode is getting much worse comparing to sync mode. Under async mode, not only the elapsed time grew a lot, but also the max queue length reached to 68487, and the memory of logger process reached about 72M. Actually, after all messages had been sent, only a few messages had been processed by logger process. Since I kept 1000 processes alive, the logger process only shared very very poor proportion CPU cycles, the message processing procedure (log to disk buffer here) became very very slow, which caused the worse elapsed time. This case can be applied on any singleton process.

On the other side, under sync mode, the max queue length did not exceed the number of simultaneous requesting processes, here is 999 in case of 1000 simultaneous processes, and the max memory of logger process is reasonable in about 674K.

I'd like to emphasize the points:

  • Singleton process in Erlang can not benefit from multiple-core scalability.
  • Move code/job out of singleton process as much as possible to simultaneous processes, for example, before send the message to singleton process, done all pre-formatting/computing job.
  • Some times, you need to sync the interacting between processes, to limit the singleton process working load.

Comments

1. Dmi -- 2009-01-20 16:00

Very nice and informative investigation! Thanks.

But... can't you to open several disk logs (even distributed) and dispatch process requests to them?

Probably by binding each process, or the object of process job to particular log?

2. Dmi -- 2009-01-20 16:00

Can't you to open several disk logs (even distributed) and dispatch process requests to them?

Probably by binding each process, or the object of process job to particular log?

3. coach factory outlet -- 2012-03-31 05:29

No one can deny the shopping at the coach factory outlet is satisfactory. For the low prices and good quality.Over the years, coach factory online has added a multitude of new handbag shapes, styles and materials to their collection. However, the highest care is taken that every Coach handbags is both aesthetically beautiful and functional.Many fashionable women match with practical Coach Purses which will make the street shopping become relaxed,and make every person can enjoy more diversiform combination in coach factory outlet online.

http://www.ccoachfactoryoutlet.com

4. louis vuitton sale -- 2012-03-31 05:30

Don't feel upset, there will be a great conversion to this kind of situation because there are a large number of louis vuitton sale now!louis vuitton outlet,welcome to buy urban louis vuitton on our online shop.discount price is our special offer, durability and high quality is our promise.louis vuitton Outlets offer famous classic brand for LV,Channel, with perfect service.So become to the VIP soon.They offer more new styles,like LV purses,LV wallets etc. And are tested by product quality monitoring center .

http://www.louisvuittonoutletsaleo.com

5. coach outlet online -- 2012-03-31 05:30

Turn your attention to such discount coach sneakers for women from coach outlet online, you will find something unique and special of such authentic coach for sale at coach factory outlet store online.coach outlet store sells goods that are constructed to meet the highest standards of quality and functionality.You can trust it 100 percent.coach outlet has become a popular shopping experience for consumers around the world, and a desirable distribution channel for manufacturer's and retailers.

http://www.coachoutletonlinesl.com

6. louis vuitton uk -- 2012-03-31 05:36

Our online store offers you discounted Designer louis vuitton replica wallet at present. You could find them in desirable quality and price. If you don't mind high class louis vuitton uk, have a good time here.These is the first time your visit our louis vuit Louis vuitton online shop ,welcome.

http://www.louisvuittonukk.org.uk

7. louis vuitton outlet -- 2012-03-31 05:37

any louis vuitton outlet New backpack features usa a function grownup overall look. Varied piece allows that it is captivated me within the approve and also further than your body.It’s induced by way of severe hardworking liver diseases, and also the most familiar factors behind constant louis vuitton bags outlet hard working liver disorder usually are excessive drinking and also liver disease Chemical.Ladies have an ardent love for the Louis Vuitton handbags outlet because Louis Vuitton enjoys a worldwide reputation of high quality and fashionable designs.

http://www.louisvuittonoutletbagsc.com

8. coach outlet store online -- 2012-03-31 05:39

coach outlet store online marketed properly all greater compared to earth and earn cozy praise from customers. They are made from the finest leather and fabric.coach outlet store have Coach handbags,Coach Shoulder Bags,Coach Briefcases and so on,these bags are so perfectly reproduced,you won't even be able to tell the difference!Coach Outlet Online Store would dynamically change your overall styles right away. The amazing knack about the unique coach handbag is that it would never disappoint your individual styles at all. Rather, it would instantly change your ultimate fashions in a remarkable manner.

http://www.coachoutletstoreonlineeo.com

9. christian louboutin uk -- 2012-04-18 01:35
10. louis vuitton uk -- 2012-04-18 01:44
11. mulberry sale -- 2012-04-18 01:52
12. anonymous -- 2012-05-21 02:00

Set against a collection that was based on a “three-dimensional” silhouette – overtly rounded sleeves, extra-wide belts, A-lines all over the place – there were only two bag styles revealed on the runway. (But I expect to see lots of different incarnations of the Box bags and <a title="Celine Handbag" href="http://www.celineonlineoutlets.com"><strong>Celine Handbag</strong></a> Luggage totes at retail stores next spring, as their popularity shows little sign of slowing down). Maybe Philo has <a title=" Celine Bag" href="http://www.celineonlineoutlets.com"><strong> Celine Bag</strong></a> gotten weary of all the attention her bags have garnered (hence the voluminous garb), but even the slight amount of bags we did see are not exciting. The geometric and yes, color-blocked envelope-style clutch (some with shoulder straps), in sharply shaped panels of suede and leather, is well, just another crisp, clean, and perfectly pleasant <a title=" Celine Luggage" href="http://www.celineonlineoutlets.com"><strong> Celine Luggage</strong></a> bag. To counter it, a round, unadorned hobo – the rustic leather – obviously the focal point of the bag – makes for good texture, but the overall effect reminds me of quintessential Coach. Clearly, it’s time for something new.

13. anonymous -- 2012-05-21 02:01

magazines and all things in the web for aggregate to do with the apple of beggarly Christian Louboutin shoes, women absolutely charge to apprentice everything. christian louboutin aperture shoes [url=http://www.christianlouboutinshoppings.com][b] christian louboutin onlineburl let you acceptable affection in your circadian life.All ladies admiring for themselves added pretty, so these humans are frequently put on Christian Louboutin [url=http://www.christianlouboutinshoppings.com][b] christian louboutin pumpsburl top heel shoes, they anticipate that all of them assume actual [url=http://www.christianlouboutinshoppings.com][b]christian louboutin outletburl abounding of [url=http://www.christianlouboutinshoppings.com][b] christian louboutin shoesburl activity and accreditation if walking and existence, christian laboutin afterwards a disbelief it's valid.

14. anonymous -- 2012-08-06 08:20

Wonderful site are display Link: online poker tips just in this blog and the great services Link: off share poker games in this blog. Thanks a lot for providing the Link: live bing oonly amazing services in this blog and Link: casino advisor online the nice services in this blog and Link: 555 gambling guide the different info.

15. anonymous -- 2012-09-17 06:05

Hit <a href= " http://cheapairjordansa.blogspot.com/ " title= " Cheap Air Jordans " ><strong>Cheap Air Jordans</strong></a>,<a href= " http://retrojordanforcheap.blogspot.com/ " title= " Retro Jordans For Cheap " ><strong>Retro Jordans For Cheap</strong></a>,<a href= " http://cheapjordan11s.blogspot.com/ " title= " Cheap Jordans 11 " ><strong>Cheap Jordans 11</strong></a>. sandwiched Baizhang long knife that went up the river Kiyoshi Xeon blow.<a href= " http://womenjordanforcheap.blogspot.com/ " title= " Womens Jordans " ><strong>Womens Jordans</strong></a>,<a href= " http://aorjordansonsale.blogspot.com/ " title= " Jordans on sale " ><strong>Jordans on sale</strong></a> blink of an eye, sword and knife column is met, the power contained in each other are all extremely Tough, the intersection point between the two, who also ; never retreat. suddenly, light flash, thunder explode Jian Qi and knife sharpening is instant, resulting in devastating explosion in the air spread fair, Jiang Kiyoshi desperate blow amazing power of the extreme, can be compared to the hidden Blade for snow, but still poor &lt; br&gt; some Fortunately, Jiang Kiyoshi very smart, her attack is divided into two parts, the first column is the sword, and the second is followed Phoenix when the sword hidden columns and snow Blade knife is met each other produce an explosion. column then the power of the sword is a powerful tool arrived, but was forced to move back to At this time, followed by Phoenix just approaching it like an invisible hand, in the back behind the river Kiyoshi one, making her forward approach. so, the snow ahead of the knife is hidden Blade suddenly meal, the two sides to form a secondary collision, resulting in a more terrible ; explosion. This time, Jiang, Kiyoshi when its red, almost Xingshenjumie. Fortunately, the magic sword to share a part of the explosion cloud force, this makes the river Kiyoshi temporarily escape death robbed. here, snow hidden Blade also been a great blow to the body shop and then shaking the tall, mouth spit out a number of Road ; blood. near the light flying, air flow, such as edge. continuous sustained devastating explosion, resulting in snow Zhenfei hidden Blade is also the spot, not badly hurt ; light field, streamer overflowing, lightning non-stop. ravioli chaotic air whistling shrill, rolling fog in the shuttle alternately. magic cloud dispersed light sword, the blade in the wind, unable to go toward the far off when Jiang Kiyoshi automatically separating from Excalibur, weakness leaves the body like the wind, his face can not see a wire color ground, Xue Feng, Ji Xue Ni, Chu Wen new face weeping, although the hearts of infinite hope, can be implemented for both sides of the crystal-clear strength &lt;br &gt; River Kiyoshi come to such a result, it is realistic for air, snow hidden Blade stand back, roaring mouth not have &lt;br &gt; The war Lu then he won out in this piece can be satisfied with the results, the hearts of the river Kiyoshi hate to death. stabilize the body, snow Hidden Blade jump into the sky, flying towards Kiyoshi chase will go, going to let her Xingshenjumie. this regard, Jiang Kiyoshi kinds of micro-eye, mouth hanging trace the vicissitudes of life a smile, she has absolutely no resistance force ; gas. a few hundred feet from the blink of </p>

16. True Religion Jeans Outlet -- 2012-10-11 06:16

Serums and mild hair sprays square measure currently turning into one among the foremost essential beauty merchandise. If you've got long hair, you wish a product that http://www.truereligionjeansoutlet-onsale.com/ True Religion Outlet controls the kink up and keeps long hair stunning and glossy. girls with crisp hair may like a reliable product to take care of the curls and convey out nice volume and bounce. you'll use your Parlux http://clarisonicmia.brushstore0o.com/ Clarisonic Outlet hand blower reception, then bring a hair blood serum or mild spray for a fast fix.

17. anonymous -- 2012-10-12 01:25

Welcome!Coach Outlet Store are loved by many people,when you walk in the street, you could see many people take Coach Shoulder Bags styles.

Coach Factory http://coachfactoryoutlet.1uxury--best.com

Coach Factory Outlet http://www.coachfactoryoutletcd.com

Coach Outlet Store http://www.coachoutletstoreusac.com

Coach Outlet http://www.coachoutletonlinecss.com

18. anonymous -- 2012-10-12 02:00

Rather than going dark and serious, Canada Goose UK has opted for fab color punches that instantly liven up the elegant ensembles.

Canada Goose Jackets http://www.canadiagoosejackets.com

Canada Goose Sale http://www.canadagoose-sale.org

Canada Goose Parkas http://www.canadagooseparkas.org

Canada Goose UK http://www.canada-goose-outlet.co.uk

19. Canada Goose Jackets -- 2012-10-12 02:01
20. Pjdjf -- 2012-10-26 01:54

More Than Just the 'Heiress'

Elie Saab has decided that his woman should be more than the “heiress” of his show’s title. He wanted to see her as <a href="http://www.monclercoat2u.com/moncler-boots-women-c-10.html"> Moncler Boots </a> “an artist, a photographer or a philanthropist.”

She was tough but tender in a mannish pantsuit trimmed with lace. At sober moments (could that be work?) she wore the new fashion <a href="http://www.monclercoat2u.com/moncler-coats-women-c-7.html">Cheap Moncler Coat </a> uniform: a soft blouse with slim skirt. But her artistic bent came through in colors like turquoise and fuchsia.

Mr. Saab, once king of red carpet and <a href="http://www.monclercoat2u.com/moncler-kids-clothing-c-11.html">Moncler Jacket For Kids </a> evening allure, had taken a more minimalist attitude, with a blue shirt and pants, <a href="http://www.monclercoat2u.com/moncler-jackets-women-c-8.html">Moncler Jackets UK </a> a shorts suit and a streamlined dress opening the show.

The designer is smart to move his fashion clock forward to daytime, while keeping a touch of glamour. Digital prints with <a href="http://www.monclercoat2u.com/moncler-kids-clothing-c-11.html">Moncler Kids</a> splashes of paint ushered in the evening wear. But even there, the focus was on slim shapes with geometric lines or with slithers of lace showing flashes of flesh. The result was a colorful but controlled collection.

21. wedding dresses 2013 -- 2012-12-27 07:43

So, just pick cute http://www.diodress.net/cheap-mother-of-the-bride-dresses-c8281/ mother of the bride dresses party dresses within our retailer instantly! Our store gives you a large surprise. Not too long ago, our keep has an activity. You'll be able to buy the discount cute celebration dresses from our on the net store. Should you http://www.diodress.net/cheap-plus-size-wedding-dresses-c8258_8270/ plus size wedding dresses tell a lot more persons the very good new, you'll get far more discount. Ladies, what exactly are you waiting for Start now to discover your great cute celebration http://www.diodress.net/cheap-beach-wedding-dresses-c8258_8264/ beach wedding dresses . In case you have any strategies, you are able to also tell us. We are going to make improvements to our service. Wish you have got a great and memorable evening!

23. anonymous -- 2013-03-27 00:02

I completely agree with you, f-commerce is about engaging potential customers and making it relevant and not always pushing product directly. Take for example Tesco Direct's latest Facebook move with the Delivery Dash Application, Engaging customers with new fun games with product at the forefront but not pushing the hard sell.jual eiger online

30. anonymous -- 2013-04-28 01:06

​@linlin http://jstlouisvuitton.webs.com/ [Louis Vuitton Outlet] ​http://christianlouboutinblogss.webs.com/ [Christian Louboutin Sale] ​http://jstjordanair.webs.com/ [Authentic Air Jordan Shoes]

31. museum art paintings -- 2013-05-11 08:02

Handmade Oil Paintings Online Gallery - MuseumArtPaintings?.com and DayDayPaint?.com. We are the best supplier of authentic hand painted art paintings. We exhibit thousands of the world’s finest oil paintings, ranging from European masterpieces to more contemporary works, offers the most comprehensive selection for you. All Oil Paintings 100% hand painted on canvas, Museum Quality with Super Low Price! <a href="http://MuseumArtPaintings.com">Photo to Oil Paintings</a>, <a href="http://daydaypaint.com">Custom Paintings from Photos</a>, Quick Turnaround! FREE SHIPPING WORLDWIDE ON ALL ORDERS! http://www.museumartpaintings.com http://www.daydaypaint.com

32. anonymous -- 2013-06-03 21:22

Your blog has some of the most fascinating information! I've read several pages here and I just had to comment and let you know that I thought you've been doing a great job here. Keep it up! tas eiger or this Saatnya total action may be usefull

33. anonymous -- 2013-06-14 02:02

Great Post. I have not been visiting the site recently. Took a visit again and there were some great comments on the site. Excellent post. Keep up the good work. Click Here - Lush Acres

34. anonymous -- 2013-06-21 05:43

It is a short drive away from Woodlands and Admiralty MRT Station. Future residents will be able to walk to the nearby Vista Point or a short drive to Causeway Point for some family fun and gatherings. A truly unique lifestyle awaits you. Woodlands EC

36. anonymous -- 2013-07-09 08:16

rare freehold condo. must check it out at Tembusu Tembusu Condo Tembusu Condominium Tembusu at Kovan

39. andre1rouse123as@… -- 2013-09-05 13:15

league of legend free riot points you can take on this site for free http://freeriotpointscodes.com/

40. anonymous -- 2013-09-05 13:17
41. anonymous -- 2013-09-05 13:17
42. wwslaxmidhar@… -- 2013-09-06 07:49

http://www.fandtlawfirm.com/ I might want to thank you for the deliberations you have invested thinking of this website. I'm trusting the same high-grade site post from you in the upcoming likewise. Actually your inventive composition aptitudes has urged me to get my own site now.

43. anonymous -- 2013-09-11 22:10

thank you for the information very helpful .. I am very interested in what you explained, it's flowing I'll try to keep track of the next post from you.If you like you can visit my website http://mhasanb.blogspot.com/2013/09/cipto-junaedy.html may be usefull

44. anonymousSDFASD -- 2013-09-28 04:50

<a href="http://www.ralphlaurenspolouk.co.uk" title="ralph lauren polo uk">ralph lauren polo uk</a> <a href="http://www.poloralphlaurensuksale.co.uk" title="polo ralph lauren uk sale">polo ralph lauren uk sale</a> <a href="http://www.ralphlaurenaustraliashirts.com" title="ralph lauren australia shirt">ralph lauren australia shirt</a> <a href="http://www.abercrombieandfitchoutletukstore.co.uk" title="abercrombie fitch outlet uk store">abercrombie fitch outlet uk store</a> <a href="http://www.franklinandmarshalloutletuk.com" title="franklin and marshall outlet uk">franklin and marshall outlet uk</a> <a href="http://www.lacostepoloukshirts.co.uk" title="lacoste polo uk shirt">lacoste polo uk shirt</a> <a href="http://www.marshallandfranklin.co.uk" title="marshall and franklin">marshall and franklin</a> <a href="http://www.edhardy-sale.com" title="ed hardy sale">ed hardy sale</a> <a href="http://www.superdryoutletstore.co.uk" title="superdry outlet store">superdry outlet store</a> <a href="http://www.ralphlaurenpoloukoutlets.co.uk" title="ralph lauren polo uk outlet">ralph lauren polo uk outlet</a> <a href="http://www.poloukoutlet.co.uk" title="polo uk outlet">polo uk outlet</a> <a href="http://www.poloralphlaurenoutlet-uk.co.uk" title="polo ralph lauren outlet uk">polo ralph lauren outlet uk</a>

45. anonymous -- 2013-10-03 23:14

Great post and a really good blog. Thank you so much for your efforts to put this amazing source of useful information together. Keep the great work up! [url=http://voltagestabilizerindonesia.com/]Jual voltage stabilizerurl [url=http://blognyakadhenk.blogspot.com/]Jasa SEO Murah Bukan Pakar SEOurl [url=http://ram-seo.com/]Jasa SEO dan Pakar SEOurl [url=http://jbvinylindo.com/]distributor vinyl beli vinyl importir vinylurl [url=http://galeriapaperindo.com/]Beli Wallpaper Toko Wallpaperurl [url=http://decorindoperkasa.com/]kayu lantai parket vinyl lantaiurl [url=http://dilajustin.blogspot.com/]Masakan Khas Indonesia Resep Masakan Resep Makanan Resep Minuman Masakan Ala Indonesiaurl [url=http://dilajustin.blogspot.com/2013/09/resep-sate-kambing-bumbu-kecap-paling.html]Resep Sate Kambing Bumbu Kecapurl [url=http://dilajustin.blogspot.com/2013/09/resep-daging-sapi-lada-hitam-empuk.html]Resep Daging Sapi Lada Hitam Empukurl [url=http://dilajustin.blogspot.com/2013/09/resep-sup-daging-lezat-gurih-enak.html]Resep Sup Daging Lezat Gurih Enakurl

46. Stefanus -- 2013-11-10 15:23

I really love your site, it give me a lot of information and knowledge, really love your site. I'll be looking for another post and will share it to my friends. Thank you so much for sharing dude. Kemeja Kerja Wanita Lengan Panjang