add support for retrieving results in multiple pages

This change improves the existing code based on the unexpected behavior reported in the following issue:  rladies#55

R/internals.R

@@ -3,26 +3,28 @@
 .quick_fetch <- function(api_url,
                          api_key = NULL,
                          event_status = NULL,
+                         offset = 0,
                          ...) {
-
+  
   # list of parameters
   parameters <- list(key = api_key,         # your api_key
                      status = event_status, # you need to add the status
                      # otherwise it will get only the upcoming event
+                     offset = offset,
                      ...                    # other parameters
   )
-
+  
   req <- httr::GET(url = api_url,          # the endpoint
                    query = parameters)
-
+  
   httr::stop_for_status(req)
   reslist <- httr::content(req, "parsed")
-
+  
   if (length(reslist) == 0) {
     stop("Zero records match your filter. Nothing to return.\n",
          call. = FALSE)
   }
-
+  
   return(list(result = reslist, headers = req$headers))
 }
 
@@ -31,45 +33,42 @@
 # Will make multiple calls to the API if needed
 # API Methods listed here: https://www.meetup.com/meetup_api/docs/
 .fetch_results <- function(api_method, api_key = NULL, event_status = NULL, ...) {
-
+  
   # Build the API endpoint URL
   meetup_api_prefix <- "https://api.meetup.com/"
   api_url <- paste0(meetup_api_prefix, api_method)
-
+  
   # Get the API key from MEETUP_KEY environment variable if NULL
   if (is.null(api_key)) api_key <- .get_api_key()
   if (!is.character(api_key)) stop("api_key must be a character string")
-
+  
   # Fetch first set of results (limited to 200 records each call)
   res <- .quick_fetch(api_url = api_url,
                       api_key = api_key,
                       event_status = event_status,
+                      offset = 0,
                       ...)
-
+  
   # Total number of records matching the query
   total_records <- as.integer(res$headers$`x-total-count`)
   if (length(total_records) == 0) total_records <- 1L
   records <- res$result
   cat(paste("Downloading", total_records, "record(s)..."))
-
-  # If you have not yet retrieved all records, calculate the # of remaining calls required
-  extra_calls <- ifelse(
-    (length(records) < total_records) & !is.null(res$headers$link),
-    floor(total_records/length(records)),
-    0)
-  if (extra_calls > 0) {
-    all_records <- list(records)
-    for (i in seq(extra_calls)) {
-      # Keep making API requests with an increasing offset value until you get all the records
-      # TO DO: clean this strsplit up or replace with regex
-
-      next_url <- strsplit(strsplit(res$headers$link, split = "<")[[1]][2], split = ">")[[1]][1]
-      res <- .quick_fetch(next_url, api_key, event_status)
-      all_records[[i + 1]] <- res$result
-    }
-    records <- unlist(all_records, recursive = FALSE)
-  }
-
+
+    if((length(records) < total_records) & !is.null(res$headers$link)){   
+      # calculate number of offsets for records above 200
+      offsetn <- ceiling(total_records/length(records))
+      all_records <- list(records)     
+      for(i in 1:(offsetn - 1)) {
+        res <- .quick_fetch(api_url = api_url,
+                            api_key = api_key,
+                            event_status = event_status,
+                            offset = i,
+                            ...)
+        all_records[[i + 1]] <- res$result
+      }
+      records <- unlist(all_records, recursive = FALSE)      
+    }  
   return(records)
 }