Added logic to handle dividing large PROGMEM strings into smaller chunks

to avoid large allocations to buffer the send.
esp8266 · mcspr · May 8, 2022 · Apr 21, 2020 · Apr 22, 2020 · Apr 22, 2020
commit 10f631198d6f2859802aff2563263e8943e8e4ef
diff --git a/libraries/DNSServer/examples/NAPTCaptivePortal/WifiHttp.h b/libraries/DNSServer/examples/NAPTCaptivePortal/WifiHttp.h
@@ -62,20 +62,7 @@ a {
   font-size: 1em;
   height: 1.8em;
 }
-/*
-*/
-</style>
-)EOF";
-#else
-static const char configHead[] PROGMEM = R"EOF(<!DOCTYPE html><html lang="en"><head><meta name='viewport' content='width=device-width'><title>WiFi</title><style>html,body{width:100%;font-family:'Open Sans',Helvetica,Arial,sans-serif;font-size:16px}h1,h2,h3,h4{vertical-align:middle;text-align:center}input[type=submit]{width:98%;font-family:'Open Sans',Helvetica,Arial,sans-serif;height:3em}a{text-decoration:none;color:blue}.mono,td,input{font-family:'Source Code Pro',monospace;height:1.5em;font-size:1em;vertical-align:middle}.lg{font-size:1em;height:1.8em}</style>)EOF";
-#endif
-
-#ifdef DEBUG_VIEW
-static const char configHead2[] PROGMEM = R"EOF(
-<style>
-/*
 
-*/
 .left {
   text-align: left;
 }
@@ -135,7 +122,7 @@ function cs(i){
 -->
 )EOF";
 #else
-static const char configHead2[] PROGMEM = R"EOF(<style>.left{text-align:left}.center{text-align:center}.right{text-align:right}#outer{width:100% overflow-y: auto}#inner{margin: .75em auto;max-width:500px;min-width:428px;text-align:left}</style></head><body onload='ol()'> <script>function ge(i){return document.getElementById(i);} function pv(){let e=ge('p');if(e.type==='password'){e.type='text';}else{e.type='password';} e.focus();} function cs(i){ge('s').value=i.innerText;let e=ge('p');e.type='password';e.value='';e.focus();e.scrollIntoView();}</script>)EOF";
+static const char configHead[] PROGMEM = R"EOF(<!DOCTYPE html><html lang="en"><head><meta name='viewport' content='width=device-width'><title>WiFi</title><style>html,body{width:100%;font-family:'Open Sans',Helvetica,Arial,sans-serif;font-size:16px}h1,h2,h3,h4{vertical-align:middle;text-align:center}input[type=submit]{width:98%;font-family:'Open Sans',Helvetica,Arial,sans-serif;height:3em}a{text-decoration:none;color:blue}.mono,td,input{font-family:'Source Code Pro',monospace;height:1.5em;font-size:1em;vertical-align:middle}.lg{font-size:1em;height:1.8em}.left{text-align:left}.center{text-align:center}.right{text-align:right}#outer{width:100% overflow-y: auto}#inner{margin: .75em auto;max-width:500px;min-width:428px;text-align:left}</style></head><body onload='ol()'> <script>function ge(i){return document.getElementById(i);} function pv(){let e=ge('p');if(e.type==='password'){e.type='text';}else{e.type='password';} e.focus();} function cs(i){ge('s').value=i.innerText;let e=ge('p');e.type='password';e.value='';e.focus();e.scrollIntoView();}</script> )EOF";
 #endif
 
 #ifdef DEBUG_VIEW
@@ -317,7 +304,6 @@ static const char configEnd[] PROGMEM = R"EOF(
 -->
 )EOF";
 #else
-static const char configEnd[] PROGMEM = R"EOF(</table> <br /><form method='POST' action='wifisave'><h4>Connect to Network:</h4> <input id='s' class='lg' type='text' size=32 maxlength=32 placeholder='Network Name/SSID' name='n' spellcheck='false' data-gramm_editor='false'/> <br /><br /> <input id='p' class='lg' type='password' size=32 maxlength=64 placeholder='password' name='p' spellcheck='false' data-gramm_editor='false'/> )EOF";
 #endif
 
 #ifdef DEBUG_VIEW
@@ -338,7 +324,7 @@ static const char configEnd2[] PROGMEM = R"EOF(
 -->
 )EOF";
 #else
-static const char configEnd2[] PROGMEM = R"EOF(&nbsp;&nbsp;<a class='lg' onclick='pv();'>&#x1f441;</a> <br /><br /><input type='submit' value='Connect/Disconnect'/></form> <br /><p>You may want to <a href='/'>return to the home page</a>.</p><p></p></div></div></body></html>)EOF";
+static const char configEnd[] PROGMEM = R"EOF(</table> <br /><form method='POST' action='wifisave'><h4>Connect to Network:</h4> <input id='s' class='lg' type='text' size=32 maxlength=32 placeholder='Network Name/SSID' name='n' spellcheck='false' data-gramm_editor='false'/> <br /><br /> <input id='p' class='lg' type='password' size=32 maxlength=64 placeholder='password' name='p' spellcheck='false' data-gramm_editor='false'/> &nbsp;&nbsp;<a class='lg' onclick='pv();'>&#x1f441;</a> <br /><br /><input type='submit' value='Connect/Disconnect'/></form> <br /><p>You may want to <a href='/'>return to the home page</a>.</p><p></p></div></div></body></html>)EOF";
 #endif
 
 #endif
diff --git a/libraries/DNSServer/examples/NAPTCaptivePortal/handleHttp.ino b/libraries/DNSServer/examples/NAPTCaptivePortal/handleHttp.ino
@@ -1,7 +1,16 @@
 #if LWIP_FEATURES && !LWIP_IPV6
 
-#include "WifiHttp.h"
-String& sendIfOver(String & str, size_t threshold = 512);
+#include "WifiHttpv2.h"
+
+#ifndef TCP_MSS
+#define TCP_MSS 1460
+#endif
+/*
+  Use kMaxChunkSize to limit size of chuncks
+*/
+constexpr size_t kMaxChunkSize = TCP_MSS;
+String& sendIfOver(String & str, size_t threshold = kMaxChunkSize/2);
+size_t sendAsChunks_P(PGM_P content, size_t chunkSize = kMaxChunkSize);
 
 size_t maxPage = 0;
 
@@ -15,18 +24,30 @@ void addNoCacheHeader() {
 String& sendIfOver(String & str, size_t threshold) {
   size_t len = str.length();
   if (len > threshold) {
+    // Use later to determine if we reserved enough room in page to avoid realloc
     maxPage = std::max(maxPage, len);
     server.sendContent(str);
     str = "";
   }
   return str;
 }
 
+/*
+  The idea here is to avoid a large allocation by sendContent_P to copy a
+  big PROGMEM string. Slice PROGMEM string into chuncks and send.
+*/
+size_t sendAsChunks_P(PGM_P content, size_t chunkSize) {
+  size_t len = strlen_P(content);
+  for (size_t pos = 0; pos < len; pos += chunkSize) {
+    server.sendContent_P(&content[pos], ((len - pos) >= chunkSize) ? chunkSize : len - pos);
+  }
+  return len;
+}
 
 /** Handle root or redirect to captive portal */
 void handleRoot() {
   if (captivePortal()) {
-    // If caprive portal is needed, redirect instead of displaying the page.
+    // If captive portal is needed, redirect instead of displaying the page.
     return;
   }
   addNoCacheHeader();
@@ -92,20 +113,19 @@ void handleWifi() {
   }
 
   // Send a few chunks of the HTML that don't need to change.
-  server.sendContent_P(configHead);
-  server.sendContent_P(configHead2);
+  sendAsChunks_P(configHead);
 
   String page;
 
-  // Just do max on some of the visually larger HTML chunks that will be loaded
-  // into page and add a little for growth when substituting values in.
-  size_t thisMany = std::max(sizeof(configWLANInfo), std::max(sizeof(configList), sizeof(configEnd))) + 200;
+  CONSOLE_PRINTLN2("sizeof(configHead): ", (sizeof(configHead)));
   CONSOLE_PRINTLN2("sizeof(configWLANInfo): ", (sizeof(configWLANInfo)));
   CONSOLE_PRINTLN2("sizeof(configList): ", (sizeof(configList)));
   CONSOLE_PRINTLN2("sizeof(configEnd): ", (sizeof(configEnd)));
-  CONSOLE_PRINTLN2("sizeof(configEnd2): ", (sizeof(configEnd2)));
-  CONSOLE_PRINTLN2("page reserve size: ", (thisMany));
-  page.reserve(thisMany);
+  // Just do max on some of the visually larger HTML chunks that will be loaded
+  // into page and add a little for growth when substituting values in.
+  size_t thisMany = std::max(sizeof(configWLANInfo), sizeof(configList)) + 200;
+  CONSOLE_PRINTLN2("Estimate Minimum page reserve size: ", (thisMany));
+  page.reserve(std::max(kMaxChunkSize, thisMany));
 
   page = FPSTR(configPresetInput);
   /*
@@ -136,6 +156,7 @@ void handleWifi() {
     page.replace("{b}", String(WiFi.softAPmacAddress()));
     page.replace("{i}", WiFi.softAPIP().toString());
     page.replace("{a}", String(sta_cnt));
+    sendIfOver(page);
     if (sta_cnt) {
       page += String(F("\r\n<PRE>\r\n"));
       struct station_info *info = wifi_softap_get_station_info();
@@ -144,6 +165,7 @@ void handleWifi() {
         addr = info->ip;
         page += macToString(info->bssid) + F("  ") + addr.toString() + F("\r\n");
         info = STAILQ_NEXT(info, next);
+        sendIfOver(page);
       }
       page += F("</PRE>\r\n");
     }
@@ -153,9 +175,9 @@ void handleWifi() {
     Before we prepare a large block for sending, we call 'sendIfOver' with a
     threshold of 0 to force the sending of the current 'page' content.
   */
-  sendIfOver(page, 0);
 
   if (WiFi.localIP().isSet()) {
+    sendIfOver(page, 0);
     page += FPSTR(configWLANInfo);
     page.replace("{s}", String(ssid));
     page.replace("{b}", macToString(bssid));
@@ -167,13 +189,12 @@ void handleWifi() {
     page.replace("{m}", WiFi.subnetMask().toString());
     page.replace("{1}", WiFi.dnsIP(0).toString());
     page.replace("{2}", WiFi.dnsIP(1).toString());
-    sendIfOver(page, 0);
   } else {
-    page += FPSTR(configWLANOffline); //F("<br /><h2>WLAN - offline</h2>");
+    page += FPSTR(configWLANOffline);
   }
 
-  page += FPSTR(configList);
   sendIfOver(page, 0);
+  sendAsChunks_P(configList);
 
   CONSOLE_PRINTLN("scan start");
   int n = WiFi.scanNetworks();
@@ -191,13 +212,9 @@ void handleWifi() {
     }
   } else {
     page += FPSTR(configNoAPs);
-    sendIfOver(page);
   }
   sendIfOver(page, 0); // send what we have buffered before next direct send.
-
-  // No changes to this chunk, no need to use Strings class. Send asis.
-  server.sendContent_P(configEnd);
-  server.sendContent_P(configEnd2);
+  sendAsChunks_P(configEnd);
 
   CONSOLE_PRINTLN2("MAX String memory used: ", (maxPage));
   server.chunkedResponseFinalize();