Merge branch 'datastore' into dev

examples/companion_radio/DataStore.cpp

@@ -0,0 +1,391 @@
+#include <Arduino.h>
+#include "DataStore.h"
+
+DataStore::DataStore(FILESYSTEM& fs, mesh::RTCClock& clock) : _fs(&fs), _clock(&clock),
+#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+    identity_store(fs, "")
+#elif defined(RP2040_PLATFORM)
+    identity_store(fs, "/identity")
+#else
+    identity_store(fs, "/identity")
+#endif
+{
+}
+
+static File openWrite(FILESYSTEM* _fs, const char* filename) {
+#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+  _fs->remove(filename);
+  return _fs->open(filename, FILE_O_WRITE);
+#elif defined(RP2040_PLATFORM)
+  return _fs->open(filename, "w");
+#else
+  return _fs->open(filename, "w", true);
+#endif
+}
+
+void DataStore::begin() {
+#if defined(RP2040_PLATFORM)
+  identity_store.begin();
+#endif
+
+#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+  checkAdvBlobFile();
+#else
+  // init 'blob store' support
+  _fs->mkdir("/bl");
+#endif
+}
+
+#if defined(ESP32)
+  #include <SPIFFS.h>
+#elif defined(RP2040_PLATFORM)
+  #include <LittleFS.h>
+#endif
+
+File DataStore::openRead(const char* filename) {
+#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+  return _fs->open(filename, FILE_O_READ);
+#elif defined(RP2040_PLATFORM)
+  return _fs->open(filename, "r");
+#else
+  return _fs->open(filename, "r", false);
+#endif
+}
+
+bool DataStore::removeFile(const char* filename) {
+  return _fs->remove(filename);
+}
+
+bool DataStore::formatFileSystem() {
+#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+  return _fs->format();
+#elif defined(RP2040_PLATFORM)
+  return LittleFS.format();
+#elif defined(ESP32)
+  return ((fs::SPIFFSFS *)_fs)->format();
+#else
+  #error "need to implement format()"
+#endif
+}
+
+bool DataStore::loadMainIdentity(mesh::LocalIdentity &identity) {
+  return identity_store.load("_main", identity);
+}
+
+bool DataStore::saveMainIdentity(const mesh::LocalIdentity &identity) {
+  return identity_store.save("_main", identity);
+}
+
+void DataStore::loadPrefs(NodePrefs& prefs, double& node_lat, double& node_lon) {
+  if (_fs->exists("/new_prefs")) {
+    loadPrefsInt("/new_prefs", prefs, node_lat, node_lon); // new filename
+  } else if (_fs->exists("/node_prefs")) {
+    loadPrefsInt("/node_prefs", prefs, node_lat, node_lon);
+    savePrefs(prefs, node_lat, node_lon);                // save to new filename
+    _fs->remove("/node_prefs"); // remove old
+  }
+}
+
+void DataStore::loadPrefsInt(const char *filename, NodePrefs& _prefs, double& node_lat, double& node_lon) {
+#if defined(RP2040_PLATFORM)
+  File file = _fs->open(filename, "r");
+#else
+  File file = _fs->open(filename);
+#endif
+  if (file) {
+    uint8_t pad[8];
+
+    file.read((uint8_t *)&_prefs.airtime_factor, sizeof(float));                           // 0
+    file.read((uint8_t *)_prefs.node_name, sizeof(_prefs.node_name));                      // 4
+    file.read(pad, 4);                                                                     // 36
+    file.read((uint8_t *)&node_lat, sizeof(node_lat));                                     // 40
+    file.read((uint8_t *)&node_lon, sizeof(node_lon));                                     // 48
+    file.read((uint8_t *)&_prefs.freq, sizeof(_prefs.freq));                               // 56
+    file.read((uint8_t *)&_prefs.sf, sizeof(_prefs.sf));                                   // 60
+    file.read((uint8_t *)&_prefs.cr, sizeof(_prefs.cr));                                   // 61
+    file.read((uint8_t *)&_prefs.reserved1, sizeof(_prefs.reserved1));                     // 62
+    file.read((uint8_t *)&_prefs.manual_add_contacts, sizeof(_prefs.manual_add_contacts)); // 63
+    file.read((uint8_t *)&_prefs.bw, sizeof(_prefs.bw));                                   // 64
+    file.read((uint8_t *)&_prefs.tx_power_dbm, sizeof(_prefs.tx_power_dbm));               // 68
+    file.read((uint8_t *)&_prefs.telemetry_mode_base, sizeof(_prefs.telemetry_mode_base)); // 69
+    file.read((uint8_t *)&_prefs.telemetry_mode_loc, sizeof(_prefs.telemetry_mode_loc));   // 70
+    file.read((uint8_t *)&_prefs.telemetry_mode_env, sizeof(_prefs.telemetry_mode_env));   // 71
+    file.read((uint8_t *)&_prefs.rx_delay_base, sizeof(_prefs.rx_delay_base));             // 72
+    file.read(pad, 4);                                                                     // 76
+    file.read((uint8_t *)&_prefs.ble_pin, sizeof(_prefs.ble_pin));                         // 80
+
+    file.close();
+  }
+}
+
+void DataStore::savePrefs(const NodePrefs& _prefs, double node_lat, double node_lon) {
+  File file = openWrite(_fs, "/new_prefs");
+  if (file) {
+    uint8_t pad[8];
+    memset(pad, 0, sizeof(pad));
+
+    file.write((uint8_t *)&_prefs.airtime_factor, sizeof(float));                           // 0
+    file.write((uint8_t *)_prefs.node_name, sizeof(_prefs.node_name));                      // 4
+    file.write(pad, 4);                                                                     // 36
+    file.write((uint8_t *)&node_lat, sizeof(node_lat));                                     // 40
+    file.write((uint8_t *)&node_lon, sizeof(node_lon));                                     // 48
+    file.write((uint8_t *)&_prefs.freq, sizeof(_prefs.freq));                               // 56
+    file.write((uint8_t *)&_prefs.sf, sizeof(_prefs.sf));                                   // 60
+    file.write((uint8_t *)&_prefs.cr, sizeof(_prefs.cr));                                   // 61
+    file.write((uint8_t *)&_prefs.reserved1, sizeof(_prefs.reserved1));                     // 62
+    file.write((uint8_t *)&_prefs.manual_add_contacts, sizeof(_prefs.manual_add_contacts)); // 63
+    file.write((uint8_t *)&_prefs.bw, sizeof(_prefs.bw));                                   // 64
+    file.write((uint8_t *)&_prefs.tx_power_dbm, sizeof(_prefs.tx_power_dbm));               // 68
+    file.write((uint8_t *)&_prefs.telemetry_mode_base, sizeof(_prefs.telemetry_mode_base)); // 69
+    file.write((uint8_t *)&_prefs.telemetry_mode_loc, sizeof(_prefs.telemetry_mode_loc));   // 70
+    file.write((uint8_t *)&_prefs.telemetry_mode_env, sizeof(_prefs.telemetry_mode_env));   // 71
+    file.write((uint8_t *)&_prefs.rx_delay_base, sizeof(_prefs.rx_delay_base));             // 72
+    file.write(pad, 4);                                                                     // 76
+    file.write((uint8_t *)&_prefs.ble_pin, sizeof(_prefs.ble_pin));                         // 80
+
+    file.close();
+  }
+}
+
+void DataStore::loadContacts(DataStoreHost* host) {
+  if (_fs->exists("/contacts3")) {
+#if defined(RP2040_PLATFORM)
+    File file = _fs->open("/contacts3", "r");
+#else
+    File file = _fs->open("/contacts3");
+#endif
+    if (file) {
+      bool full = false;
+      while (!full) {
+        ContactInfo c;
+        uint8_t pub_key[32];
+        uint8_t unused;
+
+        bool success = (file.read(pub_key, 32) == 32);
+        success = success && (file.read((uint8_t *)&c.name, 32) == 32);
+        success = success && (file.read(&c.type, 1) == 1);
+        success = success && (file.read(&c.flags, 1) == 1);
+        success = success && (file.read(&unused, 1) == 1);
+        success = success && (file.read((uint8_t *)&c.sync_since, 4) == 4); // was 'reserved'
+        success = success && (file.read((uint8_t *)&c.out_path_len, 1) == 1);
+        success = success && (file.read((uint8_t *)&c.last_advert_timestamp, 4) == 4);
+        success = success && (file.read(c.out_path, 64) == 64);
+        success = success && (file.read((uint8_t *)&c.lastmod, 4) == 4);
+        success = success && (file.read((uint8_t *)&c.gps_lat, 4) == 4);
+        success = success && (file.read((uint8_t *)&c.gps_lon, 4) == 4);
+
+        if (!success) break; // EOF
+
+        c.id = mesh::Identity(pub_key);
+        if (!host->onContactLoaded(c)) full = true;
+      }
+      file.close();
+    }
+  }
+}
+
+void DataStore::saveContacts(DataStoreHost* host) {
+  File file = openWrite(_fs, "/contacts3");
+  if (file) {
+    uint32_t idx = 0;
+    ContactInfo c;
+    uint8_t unused = 0;
+
+    while (host->getContactForSave(idx, c)) {
+      bool success = (file.write(c.id.pub_key, 32) == 32);
+      success = success && (file.write((uint8_t *)&c.name, 32) == 32);
+      success = success && (file.write(&c.type, 1) == 1);
+      success = success && (file.write(&c.flags, 1) == 1);
+      success = success && (file.write(&unused, 1) == 1);
+      success = success && (file.write((uint8_t *)&c.sync_since, 4) == 4);
+      success = success && (file.write((uint8_t *)&c.out_path_len, 1) == 1);
+      success = success && (file.write((uint8_t *)&c.last_advert_timestamp, 4) == 4);
+      success = success && (file.write(c.out_path, 64) == 64);
+      success = success && (file.write((uint8_t *)&c.lastmod, 4) == 4);
+      success = success && (file.write((uint8_t *)&c.gps_lat, 4) == 4);
+      success = success && (file.write((uint8_t *)&c.gps_lon, 4) == 4);
+
+      if (!success) break; // write failed
+
+      idx++;  // advance to next contact
+    }
+    file.close();
+  }
+}
+
+void DataStore::loadChannels(DataStoreHost* host) {
+  if (_fs->exists("/channels2")) {
+#if defined(RP2040_PLATFORM)
+    File file = _fs->open("/channels2", "r");
+#else
+    File file = _fs->open("/channels2");
+#endif
+    if (file) {
+      bool full = false;
+      uint8_t channel_idx = 0;
+      while (!full) {
+        ChannelDetails ch;
+        uint8_t unused[4];
+
+        bool success = (file.read(unused, 4) == 4);
+        success = success && (file.read((uint8_t *)ch.name, 32) == 32);
+        success = success && (file.read((uint8_t *)ch.channel.secret, 32) == 32);
+
+        if (!success) break; // EOF
+
+        if (host->onChannelLoaded(channel_idx, ch)) {
+          channel_idx++;
+        } else {
+          full = true;
+        }
+      }
+      file.close();
+    }
+  }
+}
+
+void DataStore::saveChannels(DataStoreHost* host) {
+  File file = openWrite(_fs, "/channels2");
+  if (file) {
+    uint8_t channel_idx = 0;
+    ChannelDetails ch;
+    uint8_t unused[4];
+    memset(unused, 0, 4);
+
+    while (host->getChannelForSave(channel_idx, ch)) {
+      bool success = (file.write(unused, 4) == 4);
+      success = success && (file.write((uint8_t *)ch.name, 32) == 32);
+      success = success && (file.write((uint8_t *)ch.channel.secret, 32) == 32);
+
+      if (!success) break; // write failed
+      channel_idx++;
+    }
+    file.close();
+  }
+}
+
+#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+
+#define MAX_ADVERT_PKT_LEN   (2 + 32 + PUB_KEY_SIZE + 4 + SIGNATURE_SIZE + MAX_ADVERT_DATA_SIZE)
+
+struct BlobRec {
+  uint32_t timestamp;
+  uint8_t  key[7];
+  uint8_t  len;
+  uint8_t  data[MAX_ADVERT_PKT_LEN];
+};
+
+void DataStore::checkAdvBlobFile() {
+  if (!_fs->exists("/adv_blobs")) {
+    File file = openWrite(_fs, "/adv_blobs");
+    if (file) {
+      BlobRec zeroes;
+      memset(&zeroes, 0, sizeof(zeroes));
+      for (int i = 0; i < 20; i++) {     // pre-allocate to fixed size
+        file.write((uint8_t *) &zeroes, sizeof(zeroes));
+      }
+      file.close();
+    }
+  }
+}
+
+uint8_t DataStore::getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_buf[]) {
+  File file = _fs->open("/adv_blobs");
+  uint8_t len = 0;  // 0 = not found
+
+  if (file) {
+    BlobRec tmp;
+    while (file.read((uint8_t *) &tmp, sizeof(tmp)) == sizeof(tmp)) {
+      if (memcmp(key, tmp.key, sizeof(tmp.key)) == 0) {  // only match by 7 byte prefix
+        len = tmp.len;
+        memcpy(dest_buf, tmp.data, len);
+        break;
+      }
+    }
+    file.close();
+  }
+  return len;
+}
+
+bool DataStore::putBlobByKey(const uint8_t key[], int key_len, const uint8_t src_buf[], uint8_t len) {
+  if (len < PUB_KEY_SIZE+4+SIGNATURE_SIZE || len > MAX_ADVERT_PKT_LEN) return false;
+
+  checkAdvBlobFile();
+
+  File file = _fs->open("/adv_blobs", FILE_O_WRITE);
+  if (file) {
+    uint32_t pos = 0, found_pos = 0;
+    uint32_t min_timestamp = 0xFFFFFFFF;
+
+    // search for matching key OR evict by oldest timestmap
+    BlobRec tmp;
+    file.seek(0);
+    while (file.read((uint8_t *) &tmp, sizeof(tmp)) == sizeof(tmp)) {
+      if (memcmp(key, tmp.key, sizeof(tmp.key)) == 0) {  // only match by 7 byte prefix
+        found_pos = pos;
+        break;
+      }
+      if (tmp.timestamp < min_timestamp) {
+        min_timestamp = tmp.timestamp;
+        found_pos = pos;
+      }
+
+      pos += sizeof(tmp);
+    }
+
+    memcpy(tmp.key, key, sizeof(tmp.key));  // just record 7 byte prefix of key
+    memcpy(tmp.data, src_buf, len);
+    tmp.len = len;
+    tmp.timestamp = _clock->getCurrentTime();
+
+    file.seek(found_pos);
+    file.write((uint8_t *) &tmp, sizeof(tmp));
+
+    file.close();
+    return true;
+  }
+  return false; // error
+}
+#else
+uint8_t DataStore::getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_buf[]) {
+  char path[64];
+  char fname[18];
+
+  if (key_len > 8) key_len = 8; // just use first 8 bytes (prefix)
+  mesh::Utils::toHex(fname, key, key_len);
+  sprintf(path, "/bl/%s", fname);
+
+  if (_fs->exists(path)) {
+#if defined(RP2040_PLATFORM)
+    File f = _fs->open(path, "r");
+#else
+    File f = _fs->open(path);
+#endif
+    if (f) {
+      int len = f.read(dest_buf, 255); // currently MAX 255 byte blob len supported!!
+      f.close();
+      return len;
+    }
+  }
+  return 0; // not found
+}
+
+bool DataStore::putBlobByKey(const uint8_t key[], int key_len, const uint8_t src_buf[], uint8_t len) {
+  char path[64];
+  char fname[18];
+
+  if (key_len > 8) key_len = 8; // just use first 8 bytes (prefix)
+  mesh::Utils::toHex(fname, key, key_len);
+  sprintf(path, "/bl/%s", fname);
+
+  File f = openWrite(_fs, path);
+  if (f) {
+    int n = f.write(src_buf, len);
+    f.close();
+    if (n == len) return true; // success!
+
+    _fs->remove(path); // blob was only partially written!
+  }
+  return false; // error
+}
+#endif

examples/companion_radio/DataStore.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <helpers/IdentityStore.h>
+#include <helpers/ContactInfo.h>
+#include <helpers/ChannelDetails.h>
+#include "NodePrefs.h"
+
+class DataStoreHost {
+public:
+  virtual bool onContactLoaded(const ContactInfo& contact) =0;
+  virtual bool getContactForSave(uint32_t idx, ContactInfo& contact) =0;
+  virtual bool onChannelLoaded(uint8_t channel_idx, const ChannelDetails& ch) =0;
+  virtual bool getChannelForSave(uint8_t channel_idx, ChannelDetails& ch) =0;
+};
+
+class DataStore {
+  FILESYSTEM* _fs;
+  mesh::RTCClock* _clock;
+  IdentityStore identity_store;
+
+  void loadPrefsInt(const char *filename, NodePrefs& prefs, double& node_lat, double& node_lon);
+#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+  void checkAdvBlobFile();
+#endif
+
+public:
+  DataStore(FILESYSTEM& fs, mesh::RTCClock& clock);
+  void begin();
+  bool formatFileSystem();
+  bool loadMainIdentity(mesh::LocalIdentity &identity);
+  bool saveMainIdentity(const mesh::LocalIdentity &identity);
+  void loadPrefs(NodePrefs& prefs, double& node_lat, double& node_lon);
+  void savePrefs(const NodePrefs& prefs, double node_lat, double node_lon);
+  void loadContacts(DataStoreHost* host);
+  void saveContacts(DataStoreHost* host);
+  void loadChannels(DataStoreHost* host);
+  void saveChannels(DataStoreHost* host);
+  uint8_t getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_buf[]);
+  bool putBlobByKey(const uint8_t key[], int key_len, const uint8_t src_buf[], uint8_t len);
+  File openRead(const char* filename);
+  bool removeFile(const char* filename);
+};

examples/companion_radio/MyMesh.cpp

@@ -103,200 +103,6 @@
 #include "UITask.h"
 #endif
 
-void MyMesh::loadMainIdentity() {
-  if (!_identity_store->load("_main", self_id)) {
-    self_id = radio_new_identity(); // create new random identity
-    int count = 0;
-    while (count < 10 && (self_id.pub_key[0] == 0x00 || self_id.pub_key[0] == 0xFF)) { // reserved id hashes
-      self_id = radio_new_identity();
-      count++;
-    }
-    saveMainIdentity(self_id);
-  }
-}
-
-bool MyMesh::saveMainIdentity(const mesh::LocalIdentity &identity) {
-  return _identity_store->save("_main", identity);
-}
-
-void MyMesh::loadContacts() {
-  if (_fs->exists("/contacts3")) {
-#if defined(RP2040_PLATFORM)
-    File file = _fs->open("/contacts3", "r");
-#else
-    File file = _fs->open("/contacts3");
-#endif
-    if (file) {
-      bool full = false;
-      while (!full) {
-        ContactInfo c;
-        uint8_t pub_key[32];
-        uint8_t unused;
-
-        bool success = (file.read(pub_key, 32) == 32);
-        success = success && (file.read((uint8_t *)&c.name, 32) == 32);
-        success = success && (file.read(&c.type, 1) == 1);
-        success = success && (file.read(&c.flags, 1) == 1);
-        success = success && (file.read(&unused, 1) == 1);
-        success = success && (file.read((uint8_t *)&c.sync_since, 4) == 4); // was 'reserved'
-        success = success && (file.read((uint8_t *)&c.out_path_len, 1) == 1);
-        success = success && (file.read((uint8_t *)&c.last_advert_timestamp, 4) == 4);
-        success = success && (file.read(c.out_path, 64) == 64);
-        success = success && (file.read((uint8_t *)&c.lastmod, 4) == 4);
-        success = success && (file.read((uint8_t *)&c.gps_lat, 4) == 4);
-        success = success && (file.read((uint8_t *)&c.gps_lon, 4) == 4);
-
-        if (!success) break; // EOF
-
-        c.id = mesh::Identity(pub_key);
-        if (!addContact(c)) full = true;
-      }
-      file.close();
-    }
-  }
-}
-
-void MyMesh::saveContacts() {
-#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
-  _fs->remove("/contacts3");
-  File file = _fs->open("/contacts3", FILE_O_WRITE);
-#elif defined(RP2040_PLATFORM)
-  File file = _fs->open("/contacts3", "w");
-#else
-  File file = _fs->open("/contacts3", "w", true);
-#endif
-  if (file) {
-    ContactsIterator iter;
-    ContactInfo c;
-    uint8_t unused = 0;
-
-    while (iter.hasNext(this, c)) {
-      bool success = (file.write(c.id.pub_key, 32) == 32);
-      success = success && (file.write((uint8_t *)&c.name, 32) == 32);
-      success = success && (file.write(&c.type, 1) == 1);
-      success = success && (file.write(&c.flags, 1) == 1);
-      success = success && (file.write(&unused, 1) == 1);
-      success = success && (file.write((uint8_t *)&c.sync_since, 4) == 4);
-      success = success && (file.write((uint8_t *)&c.out_path_len, 1) == 1);
-      success = success && (file.write((uint8_t *)&c.last_advert_timestamp, 4) == 4);
-      success = success && (file.write(c.out_path, 64) == 64);
-      success = success && (file.write((uint8_t *)&c.lastmod, 4) == 4);
-      success = success && (file.write((uint8_t *)&c.gps_lat, 4) == 4);
-      success = success && (file.write((uint8_t *)&c.gps_lon, 4) == 4);
-
-      if (!success) break; // write failed
-    }
-    file.close();
-  }
-}
-
-void MyMesh::loadChannels() {
-  if (_fs->exists("/channels2")) {
-#if defined(RP2040_PLATFORM)
-    File file = _fs->open("/channels2", "r");
-#else
-    File file = _fs->open("/channels2");
-#endif
-    if (file) {
-      bool full = false;
-      uint8_t channel_idx = 0;
-      while (!full) {
-        ChannelDetails ch;
-        uint8_t unused[4];
-
-        bool success = (file.read(unused, 4) == 4);
-        success = success && (file.read((uint8_t *)ch.name, 32) == 32);
-        success = success && (file.read((uint8_t *)ch.channel.secret, 32) == 32);
-
-        if (!success) break; // EOF
-
-        if (setChannel(channel_idx, ch)) {
-          channel_idx++;
-        } else {
-          full = true;
-        }
-      }
-      file.close();
-    }
-  }
-}
-
-void MyMesh::saveChannels() {
-#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
-  _fs->remove("/channels2");
-  File file = _fs->open("/channels2", FILE_O_WRITE);
-#elif defined(RP2040_PLATFORM)
-  File file = _fs->open("/channels2", "w");
-#else
-  File file = _fs->open("/channels2", "w", true);
-#endif
-  if (file) {
-    uint8_t channel_idx = 0;
-    ChannelDetails ch;
-    uint8_t unused[4];
-    memset(unused, 0, 4);
-
-    while (getChannel(channel_idx, ch)) {
-      bool success = (file.write(unused, 4) == 4);
-      success = success && (file.write((uint8_t *)ch.name, 32) == 32);
-      success = success && (file.write((uint8_t *)ch.channel.secret, 32) == 32);
-
-      if (!success) break; // write failed
-      channel_idx++;
-    }
-    file.close();
-  }
-}
-
-int MyMesh::getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_buf[]) {
-  char path[64];
-  char fname[18];
-
-  if (key_len > 8) key_len = 8; // just use first 8 bytes (prefix)
-  mesh::Utils::toHex(fname, key, key_len);
-  sprintf(path, "/bl/%s", fname);
-
-  if (_fs->exists(path)) {
-#if defined(RP2040_PLATFORM)
-    File f = _fs->open(path, "r");
-#else
-    File f = _fs->open(path);
-#endif
-    if (f) {
-      int len = f.read(dest_buf, 255); // currently MAX 255 byte blob len supported!!
-      f.close();
-      return len;
-    }
-  }
-  return 0; // not found
-}
-
-bool MyMesh::putBlobByKey(const uint8_t key[], int key_len, const uint8_t src_buf[], int len) {
-  char path[64];
-  char fname[18];
-
-  if (key_len > 8) key_len = 8; // just use first 8 bytes (prefix)
-  mesh::Utils::toHex(fname, key, key_len);
-  sprintf(path, "/bl/%s", fname);
-
-#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
-  _fs->remove(path);
-  File f = _fs->open(path, FILE_O_WRITE);
-#elif defined(RP2040_PLATFORM)
-  File f = _fs->open(path, "w");
-#else
-  File f = _fs->open(path, "w", true);
-#endif
-  if (f) {
-    int n = f.write(src_buf, len);
-    f.close();
-    if (n == len) return true; // success!
-
-    _fs->remove(path); // blob was only partially written!
-  }
-  return false; // error
-}
-
 void MyMesh::writeOKFrame() {
   uint8_t buf[1];
   buf[0] = RESP_CODE_OK;
@@ -724,14 +530,13 @@ uint32_t MyMesh::calcDirectTimeoutMillisFor(uint32_t pkt_airtime_millis, uint8_t
 
 void MyMesh::onSendTimeout() {}
 
-MyMesh::MyMesh(mesh::Radio &radio, mesh::RNG &rng, mesh::RTCClock &rtc, SimpleMeshTables &tables)
+MyMesh::MyMesh(mesh::Radio &radio, mesh::RNG &rng, mesh::RTCClock &rtc, SimpleMeshTables &tables, DataStore& store)
     : BaseChatMesh(radio, *new ArduinoMillis(), rng, rtc, *new StaticPoolPacketManager(16), tables),
-      _serial(NULL), telemetry(MAX_PACKET_PAYLOAD - 4) {
+      _serial(NULL), telemetry(MAX_PACKET_PAYLOAD - 4), _store(&store) {
   _iter_started = false;
   _cli_rescue = false;
   offline_queue_len = 0;
   app_target_ver = 0;
-  _identity_store = NULL;
   pending_login = pending_status = pending_telemetry = 0;
   next_ack_idx = 0;
   sign_data = NULL;
@@ -749,62 +554,18 @@ MyMesh::MyMesh(mesh::Radio &radio, mesh::RNG &rng, mesh::RTCClock &rtc, SimpleMe
   //_prefs.rx_delay_base = 10.0f;  enable once new algo fixed
 }
 
-void MyMesh::loadPrefsInt(const char *filename) {
-#if defined(RP2040_PLATFORM)
-  File file = _fs->open(filename, "r");
-#else
-  File file = _fs->open(filename);
-#endif
-  if (file) {
-    uint8_t pad[8];
-
-    file.read((uint8_t *)&_prefs.airtime_factor, sizeof(float));                           // 0
-    file.read((uint8_t *)_prefs.node_name, sizeof(_prefs.node_name));                      // 4
-    file.read(pad, 4);                                                                     // 36
-    file.read((uint8_t *)&sensors.node_lat, sizeof(sensors.node_lat));                     // 40
-    file.read((uint8_t *)&sensors.node_lon, sizeof(sensors.node_lon));                     // 48
-    file.read((uint8_t *)&_prefs.freq, sizeof(_prefs.freq));                               // 56
-    file.read((uint8_t *)&_prefs.sf, sizeof(_prefs.sf));                                   // 60
-    file.read((uint8_t *)&_prefs.cr, sizeof(_prefs.cr));                                   // 61
-    file.read((uint8_t *)&_prefs.reserved1, sizeof(_prefs.reserved1));                     // 62
-    file.read((uint8_t *)&_prefs.manual_add_contacts, sizeof(_prefs.manual_add_contacts)); // 63
-    file.read((uint8_t *)&_prefs.bw, sizeof(_prefs.bw));                                   // 64
-    file.read((uint8_t *)&_prefs.tx_power_dbm, sizeof(_prefs.tx_power_dbm));               // 68
-    file.read((uint8_t *)&_prefs.telemetry_mode_base, sizeof(_prefs.telemetry_mode_base)); // 69
-    file.read((uint8_t *)&_prefs.telemetry_mode_loc, sizeof(_prefs.telemetry_mode_loc));   // 70
-    file.read((uint8_t *)&_prefs.telemetry_mode_env, sizeof(_prefs.telemetry_mode_env));   // 71
-    file.read((uint8_t *)&_prefs.rx_delay_base, sizeof(_prefs.rx_delay_base));             // 72
-    file.read(pad, 4);                                                                     // 76
-    file.read((uint8_t *)&_prefs.ble_pin, sizeof(_prefs.ble_pin));                         // 80
-
-    // sanitise bad pref values
-    _prefs.rx_delay_base = constrain(_prefs.rx_delay_base, 0, 20.0f);
-    _prefs.airtime_factor = constrain(_prefs.airtime_factor, 0, 9.0f);
-    _prefs.freq = constrain(_prefs.freq, 400.0f, 2500.0f);
-    _prefs.bw = constrain(_prefs.bw, 62.5f, 500.0f);
-    _prefs.sf = constrain(_prefs.sf, 7, 12);
-    _prefs.cr = constrain(_prefs.cr, 5, 8);
-    _prefs.tx_power_dbm = constrain(_prefs.tx_power_dbm, 1, MAX_LORA_TX_POWER);
-
-    file.close();
-  }
-}
-
-void MyMesh::begin(FILESYSTEM &fs, bool has_display) {
-  _fs = &fs;
-
+void MyMesh::begin(bool has_display) {
   BaseChatMesh::begin();
 
-#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
-  _identity_store = new IdentityStore(fs, "");
-#elif defined(RP2040_PLATFORM)
-  _identity_store = new IdentityStore(fs, "/identity");
-  _identity_store->begin();
-#else
-  _identity_store = new IdentityStore(fs, "/identity");
-#endif
-
-  loadMainIdentity();
+  if (!_store->loadMainIdentity(self_id)) {
+    self_id = radio_new_identity(); // create new random identity
+    int count = 0;
+    while (count < 10 && (self_id.pub_key[0] == 0x00 || self_id.pub_key[0] == 0xFF)) { // reserved id hashes
+      self_id = radio_new_identity();
+      count++;
+    }
+    _store->saveMainIdentity(self_id);
+  }
 
   // use hex of first 4 bytes of identity public key as default node name
   char pub_key_hex[10];
@@ -817,13 +578,16 @@ void MyMesh::begin(FILESYSTEM &fs, bool has_display) {
 #endif
 
   // load persisted prefs
-  if (_fs->exists("/new_prefs")) {
-    loadPrefsInt("/new_prefs"); // new filename
-  } else if (_fs->exists("/node_prefs")) {
-    loadPrefsInt("/node_prefs");
-    savePrefs();                // save to new filename
-    _fs->remove("/node_prefs"); // remove old
-  }
+  _store->loadPrefs(_prefs, sensors.node_lat, sensors.node_lon);
+
+  // sanitise bad pref values
+  _prefs.rx_delay_base = constrain(_prefs.rx_delay_base, 0, 20.0f);
+  _prefs.airtime_factor = constrain(_prefs.airtime_factor, 0, 9.0f);
+  _prefs.freq = constrain(_prefs.freq, 400.0f, 2500.0f);
+  _prefs.bw = constrain(_prefs.bw, 62.5f, 500.0f);
+  _prefs.sf = constrain(_prefs.sf, 7, 12);
+  _prefs.cr = constrain(_prefs.cr, 5, 8);
+  _prefs.tx_power_dbm = constrain(_prefs.tx_power_dbm, 1, MAX_LORA_TX_POWER);
 
 #ifdef BLE_PIN_CODE
   if (_prefs.ble_pin == 0) {
@@ -844,12 +608,9 @@ void MyMesh::begin(FILESYSTEM &fs, bool has_display) {
   _active_ble_pin = 0;
 #endif
 
-  // init 'blob store' support
-  _fs->mkdir("/bl");
-
-  loadContacts();
+  _store->loadContacts(this);
   addChannel("Public", PUBLIC_GROUP_PSK); // pre-configure Andy's public channel
-  loadChannels();
+  _store->loadChannels(this);
 
   radio_set_params(_prefs.freq, _prefs.bw, _prefs.sf, _prefs.cr);
   radio_set_tx_power(_prefs.tx_power_dbm);
@@ -870,42 +631,6 @@ void MyMesh::startInterface(BaseSerialInterface &serial) {
   serial.enable();
 }
 
-void MyMesh::savePrefs() {
-#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
-  _fs->remove("/new_prefs");
-  File file = _fs->open("/new_prefs", FILE_O_WRITE);
-#elif defined(RP2040_PLATFORM)
-  File file = _fs->open("/new_prefs", "w");
-#else
-  File file = _fs->open("/new_prefs", "w", true);
-#endif
-  if (file) {
-    uint8_t pad[8];
-    memset(pad, 0, sizeof(pad));
-
-    file.write((uint8_t *)&_prefs.airtime_factor, sizeof(float));                           // 0
-    file.write((uint8_t *)_prefs.node_name, sizeof(_prefs.node_name));                      // 4
-    file.write(pad, 4);                                                                     // 36
-    file.write((uint8_t *)&sensors.node_lat, sizeof(sensors.node_lat));                     // 40
-    file.write((uint8_t *)&sensors.node_lon, sizeof(sensors.node_lon));                     // 48
-    file.write((uint8_t *)&_prefs.freq, sizeof(_prefs.freq));                               // 56
-    file.write((uint8_t *)&_prefs.sf, sizeof(_prefs.sf));                                   // 60
-    file.write((uint8_t *)&_prefs.cr, sizeof(_prefs.cr));                                   // 61
-    file.write((uint8_t *)&_prefs.reserved1, sizeof(_prefs.reserved1));                     // 62
-    file.write((uint8_t *)&_prefs.manual_add_contacts, sizeof(_prefs.manual_add_contacts)); // 63
-    file.write((uint8_t *)&_prefs.bw, sizeof(_prefs.bw));                                   // 64
-    file.write((uint8_t *)&_prefs.tx_power_dbm, sizeof(_prefs.tx_power_dbm));               // 68
-    file.write((uint8_t *)&_prefs.telemetry_mode_base, sizeof(_prefs.telemetry_mode_base)); // 69
-    file.write((uint8_t *)&_prefs.telemetry_mode_loc, sizeof(_prefs.telemetry_mode_loc));   // 70
-    file.write((uint8_t *)&_prefs.telemetry_mode_env, sizeof(_prefs.telemetry_mode_env));   // 71
-    file.write((uint8_t *)&_prefs.rx_delay_base, sizeof(_prefs.rx_delay_base));             // 72
-    file.write(pad, 4);                                                                     // 76
-    file.write((uint8_t *)&_prefs.ble_pin, sizeof(_prefs.ble_pin));                         // 80
-
-    file.close();
-  }
-}
-
 void MyMesh::handleCmdFrame(size_t len) {
   if (cmd_frame[0] == CMD_DEVICE_QEURY && len >= 2) { // sent when app establishes connection
     app_target_ver = cmd_frame[1];                    // which version of protocol does app understand
@@ -1286,7 +1011,7 @@ void MyMesh::handleCmdFrame(size_t len) {
 #if ENABLE_PRIVATE_KEY_IMPORT
     mesh::LocalIdentity identity;
     identity.readFrom(&cmd_frame[1], 64);
-    if (saveMainIdentity(identity)) {
+    if (_store->saveMainIdentity(identity)) {
       self_id = identity;
       writeOKFrame();
     } else {
@@ -1536,19 +1261,6 @@ void MyMesh::enterCLIRescue() {
   Serial.println("========= CLI Rescue =========");
 }
 
-bool MyMesh::formatFileSystem() {
-#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
-    return InternalFS.format();
-#elif defined(RP2040_PLATFORM)
-    return LittleFS.format();
-#elif defined(ESP32)
-    return SPIFFS.format();
-#else
-    #error "need to implement file system erase"
-    return false;
-#endif
-}
-
 void MyMesh::checkCLIRescueCmd() {
   int len = strlen(cli_command);
   while (Serial.available() && len < sizeof(cli_command)-1) {
@@ -1576,21 +1288,89 @@ void MyMesh::checkCLIRescueCmd() {
         Serial.printf("  Error: unknown config: %s\n", config);
       }
     } else if (strcmp(cli_command, "rebuild") == 0) {
-      bool success = formatFileSystem();
+      bool success = _store->formatFileSystem();
       if (success) {
-        saveMainIdentity(self_id);
+        _store->saveMainIdentity(self_id);
+        savePrefs();
         saveContacts();
+        saveChannels();
         Serial.println("  > erase and rebuild done");
       } else {
         Serial.println("  Error: erase failed");
       }
     } else if (strcmp(cli_command, "erase") == 0) {
-      bool success = formatFileSystem();
+      bool success = _store->formatFileSystem();
       if (success) {
         Serial.println("  > erase done");
       } else {
         Serial.println("  Error: erase failed");
       }
+    } else if (memcmp(cli_command, "ls", 2) == 0) {
+
+      // get path from command e.g: "ls /adafruit"
+      const char *path = &cli_command[3];
+      
+      // log each file and directory
+      File root = _store->openRead(path);
+      if(root){
+        File file = root.openNextFile();
+        while (file) {
+
+          if (file.isDirectory()) {
+            Serial.printf("[dir] %s\n", file.name());
+          } else {
+            Serial.printf("[file] %s (%d bytes)\n", file.name(), file.size());
+          }
+
+          // move to next file
+          file = root.openNextFile();
+
+        }
+        root.close();
+      }
+
+    } else if (memcmp(cli_command, "cat", 3) == 0) {
+
+      // get path from command e.g: "cat /contacts3"
+      const char *path = &cli_command[4];
+      
+      // log file content as hex
+      File file = _store->openRead(path);
+      if(file){
+
+        // get file content
+        int file_size = file.available();
+        uint8_t buffer[file_size];
+        file.read(buffer, file_size);
+
+        // print hex
+        mesh::Utils::printHex(Serial, buffer, file_size);
+        Serial.print("\n");
+
+        file.close();
+
+      }
+
+    } else if (memcmp(cli_command, "rm ", 3) == 0) {
+
+      // get path from command e.g: "rm /adv_blobs"
+      const char *path = &cli_command[4];
+
+      // ensure path is not empty, or root dir
+      if(!path || strlen(path) == 0 || strcmp(path, "/") == 0){
+        Serial.println("Invalid path provided");
+      } else {
+
+        // remove file
+        bool removed = _store->removeFile(path);
+        if(removed){
+          Serial.println("File removed");
+        } else {
+          Serial.println("Failed to remove file");
+        }
+
+      }
+
     } else if (strcmp(cli_command, "reboot") == 0) {
       board.reboot();  // doesn't return
     } else {

examples/companion_radio/MyMesh.h

@@ -25,6 +25,7 @@
 #include <SPIFFS.h>
 #endif
 
+#include "DataStore.h"
 #include "NodePrefs.h"
 
 #include <RTClib.h>
@@ -76,14 +77,12 @@
 #define REQ_TYPE_KEEP_ALIVE             0x02
 #define REQ_TYPE_GET_TELEMETRY_DATA     0x03
 
-class MyMesh : public BaseChatMesh {
+class MyMesh : public BaseChatMesh, public DataStoreHost {
 public:
-  MyMesh(mesh::Radio &radio, mesh::RNG &rng, mesh::RTCClock &rtc, SimpleMeshTables &tables);
+  MyMesh(mesh::Radio &radio, mesh::RNG &rng, mesh::RTCClock &rtc, SimpleMeshTables &tables, DataStore& store);
 
-  void begin(FILESYSTEM &fs, bool has_display);
+  void begin(bool has_display);
   void startInterface(BaseSerialInterface &serial);
-  void loadPrefsInt(const char *filename);
-  void savePrefs();
 
   const char *getNodeName();
   NodePrefs *getNodePrefs();
@@ -127,6 +126,12 @@ protected:
   uint32_t calcDirectTimeoutMillisFor(uint32_t pkt_airtime_millis, uint8_t path_len) const override;
   void onSendTimeout() override;
 
+  // DataStoreHost methods
+  bool onContactLoaded(const ContactInfo& contact) override { return addContact(contact); }
+  bool getContactForSave(uint32_t idx, ContactInfo& contact) override { return getContactByIdx(idx, contact); }
+  bool onChannelLoaded(uint8_t channel_idx, const ChannelDetails& ch) override { return setChannel(channel_idx, ch); }
+  bool getChannelForSave(uint8_t channel_idx, ChannelDetails& ch) override { return getChannel(channel_idx, ch); }
+
 private:
   void writeOKFrame();
   void writeErrFrame(uint8_t err_code);
@@ -135,22 +140,23 @@ private:
   void updateContactFromFrame(ContactInfo &contact, const uint8_t *frame, int len);
   void addToOfflineQueue(const uint8_t frame[], int len);
   int getFromOfflineQueue(uint8_t frame[]);
-  void loadMainIdentity();
-  bool saveMainIdentity(const mesh::LocalIdentity &identity);
-  void loadContacts();
-  void saveContacts();
-  void loadChannels();
-  void saveChannels();
-  int getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_buf[]) override;
-  bool putBlobByKey(const uint8_t key[], int key_len, const uint8_t src_buf[], int len) override;
+  int getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_buf[]) override { 
+    return _store->getBlobByKey(key, key_len, dest_buf);
+  }
+  bool putBlobByKey(const uint8_t key[], int key_len, const uint8_t src_buf[], int len) override {
+    return _store->putBlobByKey(key, key_len, src_buf, len);
+  }
 
   void checkCLIRescueCmd();
   void checkSerialInterface();
-  bool formatFileSystem();
+
+  // helpers, short-cuts
+  void savePrefs() { _store->savePrefs(_prefs, sensors.node_lat, sensors.node_lon); }
+  void saveChannels() { _store->saveChannels(this); }
+  void saveContacts() { _store->saveContacts(this); }
 
 private:
-  FILESYSTEM *_fs;
-  IdentityStore *_identity_store;
+  DataStore* _store;
   NodePrefs _prefs;
   uint32_t pending_login;
   uint32_t pending_status;

examples/companion_radio/main.cpp

@@ -14,10 +14,13 @@ static uint32_t _atoi(const char* sp) {
 
 #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
   #include <InternalFileSystem.h>
+  DataStore store(InternalFS, rtc_clock);
 #elif defined(RP2040_PLATFORM)
   #include <LittleFS.h>
+  DataStore store(LittleFS, rtc_clock);
 #elif defined(ESP32)
   #include <SPIFFS.h>
+  DataStore store(SPIFFS, rtc_clock);
 #endif
 
 #ifdef ESP32
@@ -74,7 +77,7 @@ static uint32_t _atoi(const char* sp) {
 /* GLOBAL OBJECTS */
 StdRNG fast_rng;
 SimpleMeshTables tables;
-MyMesh the_mesh(radio_driver, fast_rng, rtc_clock, tables); 
+MyMesh the_mesh(radio_driver, fast_rng, rtc_clock, tables, store);
 
 #ifdef DISPLAY_CLASS
   #include "UITask.h"
@@ -82,7 +85,6 @@ MyMesh the_mesh(radio_driver, fast_rng, rtc_clock, tables);
 #endif
 /* END GLOBAL OBJECTS */
 
-
 void halt() {
   while (1) ;
 }
@@ -108,7 +110,8 @@ void setup() {
 
 #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
   InternalFS.begin();
-  the_mesh.begin(InternalFS,
+  store.begin();
+  the_mesh.begin(
     #ifdef DISPLAY_CLASS
         disp != NULL
     #else
@@ -126,7 +129,8 @@ void setup() {
   the_mesh.startInterface(serial_interface);
 #elif defined(RP2040_PLATFORM)
   LittleFS.begin();
-  the_mesh.begin(LittleFS,
+  store.begin();
+  the_mesh.begin(
     #ifdef DISPLAY_CLASS
         disp != NULL
     #else
@@ -151,7 +155,8 @@ void setup() {
     the_mesh.startInterface(serial_interface);
 #elif defined(ESP32)
   SPIFFS.begin(true);
-  the_mesh.begin(SPIFFS,
+  store.begin();
+  the_mesh.begin(
     #ifdef DISPLAY_CLASS
         disp != NULL
     #else

src/helpers/BaseChatMesh.cpp

@@ -699,6 +699,13 @@ int BaseChatMesh::findChannelIdx(const mesh::GroupChannel& ch) {
 }
 #endif
 
+bool BaseChatMesh::getContactByIdx(uint32_t idx, ContactInfo& contact) {
+  if (idx >= num_contacts) return false;
+
+  contact = contacts[idx];
+  return true;
+}
+
 ContactsIterator BaseChatMesh::startContactsIterator() {
   return ContactsIterator();
 }

src/helpers/BaseChatMesh.h

@@ -7,19 +7,7 @@
 
 #define MAX_TEXT_LEN    (10*CIPHER_BLOCK_SIZE)  // must be LESS than (MAX_PACKET_PAYLOAD - 4 - CIPHER_MAC_SIZE - 1)
 
-struct ContactInfo {
-  mesh::Identity id;
-  char name[32];
-  uint8_t type;   // on of ADV_TYPE_*
-  uint8_t flags;
-  int8_t out_path_len;
-  uint8_t out_path[MAX_PATH_SIZE];
-  uint32_t last_advert_timestamp;   // by THEIR clock
-  uint8_t shared_secret[PUB_KEY_SIZE];
-  uint32_t lastmod;  // by OUR clock
-  int32_t gps_lat, gps_lon;    // 6 dec places
-  uint32_t sync_since;
-};
+#include "ContactInfo.h"
 
 #define MAX_SEARCH_RESULTS   8
 
@@ -61,10 +49,7 @@ struct ConnectionInfo {
   uint32_t expected_ack;
 };
 
-struct ChannelDetails {
-  mesh::GroupChannel channel;
-  char name[32];
-};
+#include "ChannelDetails.h"
 
 /**
  *  \brief  abstract Mesh class for common 'chat' client
@@ -158,6 +143,7 @@ public:
   bool  removeContact(ContactInfo& contact);
   bool  addContact(const ContactInfo& contact);
   int getNumContacts() const { return num_contacts; }
+  bool getContactByIdx(uint32_t idx, ContactInfo& contact);
   ContactsIterator startContactsIterator();
   ChannelDetails* addChannel(const char* name, const char* psk_base64);
   bool getChannel(int idx, ChannelDetails& dest);

src/helpers/ChannelDetails.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <Arduino.h>
+#include <Mesh.h>
+
+struct ChannelDetails {
+  mesh::GroupChannel channel;
+  char name[32];
+};

src/helpers/ContactInfo.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <Arduino.h>
+#include <Mesh.h>
+
+struct ContactInfo {
+  mesh::Identity id;
+  char name[32];
+  uint8_t type;   // on of ADV_TYPE_*
+  uint8_t flags;
+  int8_t out_path_len;
+  uint8_t out_path[MAX_PATH_SIZE];
+  uint32_t last_advert_timestamp;   // by THEIR clock
+  uint8_t shared_secret[PUB_KEY_SIZE];
+  uint32_t lastmod;  // by OUR clock
+  int32_t gps_lat, gps_lon;    // 6 dec places
+  uint32_t sync_since;
+};

variants/t114/platformio.ini

@@ -70,7 +70,7 @@ build_flags =
   -D MAX_CONTACTS=100
   -D MAX_GROUP_CHANNELS=8
   -D BLE_PIN_CODE=123456
-  -D BLE_DEBUG_LOGGING=1
+;  -D BLE_DEBUG_LOGGING=1
   -D OFFLINE_QUEUE_SIZE=256
 ;  -D MESH_PACKET_LOGGING=1
 ;  -D MESH_DEBUG=1