[Commits] [SCM] claws branch, master, updated. 3.15.0-147-g9729bf0
ticho at claws-mail.org
ticho at claws-mail.org
Mon Oct 23 13:51:11 CEST 2017
The branch, master has been updated
via 9729bf0e0d39ca60ef9717d7f059e4dfdf7fd60d (commit)
from 126ff4dcbc912d839ba9e0340857d22abf71e0ef (commit)
Summary of changes:
src/etpan/nntp-thread.c | 124 ++++++++++++++++++++++++++++++++++++++---------
src/news.c | 4 +-
2 files changed, 103 insertions(+), 25 deletions(-)
- Log -----------------------------------------------------------------
commit 9729bf0e0d39ca60ef9717d7f059e4dfdf7fd60d
Author: Andrej Kacian <ticho at claws-mail.org>
Date: Mon Oct 23 13:49:15 2017 +0200
Fetch XOVER and XHDR data in batches of 5000.
This avoids long stalls, which sometimes confuse libetpan.
It also allows implementing some sort of progressbar in UI.
diff --git a/src/etpan/nntp-thread.c b/src/etpan/nntp-thread.c
index 8ebce0d..fb2cdfa 100644
--- a/src/etpan/nntp-thread.c
+++ b/src/etpan/nntp-thread.c
@@ -51,6 +51,8 @@
#define DISABLE_LOG_DURING_LOGIN
+#define NNTP_BATCH_SIZE 4999
+
static struct etpan_thread_manager * thread_manager = NULL;
static chash * nntp_hash = NULL;
static chash * session_hash = NULL;
@@ -822,26 +824,62 @@ static void xover_run(struct etpan_thread_op * op)
}
result->error = r;
- debug_print("nntp xover run - end %i\n", r);
+ debug_print("nntp xover run %d-%d - end %i\n",
+ param->beg, param->end, r);
}
int nntp_threaded_xover(Folder * folder, guint32 beg, guint32 end, struct newsnntp_xover_resp_item **single_result, clist **multiple_result)
{
struct xover_param param;
struct xover_result result;
-
- debug_print("nntp xover - begin\n");
-
- param.nntp = get_nntp(folder);
- param.beg = beg;
- param.end = end;
- param.result = single_result;
- param.msglist = multiple_result;
-
- threaded_run(folder, ¶m, &result, xover_run);
+ clist *l = NULL, *h = NULL;
+ guint32 cbeg = 0, cend = 0;
+
+ debug_print("nntp xover - begin (%d-%d)\n", beg, end);
+
+ h = clist_new();
+
+ /* Request the overview in batches of NNTP_BATCH_SIZE, to prevent
+ * long stalls or libetpan choking on too large server response,
+ * and to allow updating any progress indicators while we work. */
+ cbeg = beg;
+ while (cbeg <= end && cend <= end) {
+ cend = cbeg + NNTP_BATCH_SIZE;
+ if (cend > end)
+ cend = end;
+
+ param.nntp = get_nntp(folder);
+ param.beg = cbeg;
+ param.end = cend;
+ param.result = single_result;
+ param.msglist = &l;
+
+ threaded_run(folder, ¶m, &result, xover_run);
+
+ /* Handle errors */
+ if (result.error != NEWSNNTP_NO_ERROR) {
+ log_warning(LOG_PROTOCOL, _("couldn't get xover range\n"));
+ debug_print("couldn't get xover for %d-%d\n", cbeg, cend);
+ if (l != NULL)
+ clist_free(l);
+ return result.error;
+ }
+
+ /* Append the new data (l) to list of results (h). */
+ if (l != NULL) {
+ debug_print("total items so far %d, items this batch %d\n",
+ clist_count(h), clist_count(l));
+ clist_concat(h, l);
+ l = NULL;
+ }
+
+ cbeg += NNTP_BATCH_SIZE + 1;
+ }
debug_print("nntp xover - end\n");
-
+
+ *multiple_result = h;
+
return result.error;
}
@@ -875,26 +913,64 @@ static void xhdr_run(struct etpan_thread_op * op)
}
result->error = r;
- debug_print("nntp xhdr run - end %i\n", r);
+ debug_print("nntp xhdr '%s %d-%d' run - end %i\n",
+ param->header, param->beg, param->end, r);
}
int nntp_threaded_xhdr(Folder * folder, const char *header, guint32 beg, guint32 end, clist **hdrlist)
{
struct xhdr_param param;
struct xhdr_result result;
+ clist *l = NULL;
+ clist *h = *hdrlist;
+ guint32 cbeg = 0, cend = 0;
+
+ debug_print("nntp xhdr %s - begin (%d-%d)\n", header, beg, end);
+
+ if (h == NULL)
+ h = clist_new();
+
+ /* Request the headers in batches of NNTP_BATCH_SIZE, to prevent
+ * long stalls or libetpan choking on too large server response,
+ * and to allow updating any progress indicators while we work. */
+ cbeg = beg;
+ while (cbeg <= end && cend <= end) {
+ cend = cbeg + NNTP_BATCH_SIZE;
+ if (cend > end)
+ cend = end;
+
+ param.nntp = get_nntp(folder);
+ param.header = header;
+ param.beg = cbeg;
+ param.end = cend;
+ param.hdrlist = &l;
+
+ threaded_run(folder, ¶m, &result, xhdr_run);
+
+ /* Handle errors */
+ if (result.error != NEWSNNTP_NO_ERROR) {
+ log_warning(LOG_PROTOCOL, _("couldn't get xhdr range\n"));
+ debug_print("couldn't get xhdr %s %d-%d\n", header, cbeg, cend);
+ if (l != NULL)
+ clist_free(l);
+ return result.error;
+ }
+
+ /* Append the new data (l) to list of results (h). */
+ if (l != NULL) {
+ debug_print("total items so far %d, items this batch %d\n",
+ clist_count(h), clist_count(l));
+ clist_concat(h, l);
+ l = NULL;
+ }
+
+ cbeg += NNTP_BATCH_SIZE + 1;
+ }
- debug_print("nntp xhdr - begin\n");
-
- param.nntp = get_nntp(folder);
- param.header = header;
- param.beg = beg;
- param.end = end;
- param.hdrlist = hdrlist;
+ debug_print("nntp xhdr %s - end (%d-%d)\n", header, beg, end);
+
+ *hdrlist = h;
- threaded_run(folder, ¶m, &result, xhdr_run);
-
- debug_print("nntp xhdr - end\n");
-
return result.error;
}
diff --git a/src/news.c b/src/news.c
index 720b069..ac59f0c 100644
--- a/src/news.c
+++ b/src/news.c
@@ -1149,6 +1149,7 @@ static void news_get_extra_fields(NewsSession *session, FolderItem *item, GSList
}
}
newsnntp_xhdr_free(hdrlist);
+ hdrlist = NULL;
/* To */
ok = nntp_threaded_xhdr(item->folder, "to", first, last, &hdrlist);
@@ -1173,6 +1174,7 @@ static void news_get_extra_fields(NewsSession *session, FolderItem *item, GSList
}
}
newsnntp_xhdr_free(hdrlist);
+ hdrlist = NULL;
/* Cc */
ok = nntp_threaded_xhdr(item->folder, "cc", first, last, &hdrlist);
@@ -1197,7 +1199,7 @@ static void news_get_extra_fields(NewsSession *session, FolderItem *item, GSList
}
}
newsnntp_xhdr_free(hdrlist);
-
+ hdrlist = NULL;
g_hash_table_destroy(hash_table);
news_folder_unlock(NEWS_FOLDER(item->folder));
-----------------------------------------------------------------------
hooks/post-receive
--
Claws Mail
More information about the Commits
mailing list