rpm: Handle proper return value to avoid major issues

0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch changed
to avoid critical issues
Handled return values of getrlimit() and lzma_cputhreads() functions
to avoid unexpected behaviours like devide by zero and potential read
of uninitialized variable 'virtual_memory'
Upstream-Status: Pending [merge of multithreading patches to upstream]

Signed-off-by: Ranjitsinh Rathod <ranjitsinh.rathod@kpit.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit 5aae9c2cb464350bc443a0f60fd6602942e61f46)
Signed-off-by: Steve Sakoman <steve@sakoman.com>

diff --git a/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch b/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch
index 64547852548a8b2015b4e9b59b27857a5e6fb152..dc3f74fecd19579038aa5aacdee43abd4a44eaa3 100644 (file)
--- a/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch
+++ b/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch
@@ -11,36 +11,39 @@ CPU thread.
 Upstream-Status: Pending [merge of multithreading patches to upstream]
 
 Signed-off-by: Peter Bergin <peter@berginkonsult.se>
+Signed-off-by: Ranjitsinh Rathod <ranjitsinh.rathod@kpit.com>
 ---
- rpmio/rpmio.c | 34 ++++++++++++++++++++++++++++++++++
- 1 file changed, 34 insertions(+)
+ rpmio/rpmio.c | 36 ++++++++++++++++++++++++++++++++++++
+ 1 file changed, 36 insertions(+)
 
 diff --git a/rpmio/rpmio.c b/rpmio/rpmio.c
 index e051c98..b3c56b6 100644
 --- a/rpmio/rpmio.c
 +++ b/rpmio/rpmio.c
-@@ -845,6 +845,40 @@ static LZFILE *lzopen_internal(const char *mode, int fd, int xz)
+@@ -845,6 +845,42 @@ static LZFILE *lzopen_internal(const char *mode, int fd, int xz)
                }
  #endif
  
-+              struct rlimit virtual_memory;
-+              getrlimit(RLIMIT_AS, &virtual_memory);
-+              if (virtual_memory.rlim_cur != RLIM_INFINITY) {
++              struct rlimit virtual_memory = {RLIM_INFINITY , RLIM_INFINITY};
++              int status = getrlimit(RLIMIT_AS, &virtual_memory);
++              if ((status != -1) && (virtual_memory.rlim_cur != RLIM_INFINITY)) {
 +                      const uint64_t virtual_memlimit = virtual_memory.rlim_cur;
++                      uint32_t threads_max = lzma_cputhreads();
 +                      const uint64_t virtual_memlimit_per_cpu_thread =
-+                              virtual_memlimit / lzma_cputhreads();
-+                      uint64_t memory_usage_virt;
++                              virtual_memlimit / ((threads_max == 0) ? 1 : threads_max);
 +                      rpmlog(RPMLOG_NOTICE, "XZ: virtual memory restricted to %lu and "
 +                             "per CPU thread %lu\n", virtual_memlimit, virtual_memlimit_per_cpu_thread);
++                      uint64_t memory_usage_virt;
 +                      /* keep reducing the number of compression threads until memory
 +                         usage falls below the limit per CPU thread*/
 +                      while ((memory_usage_virt = lzma_stream_encoder_mt_memusage(&mt_options)) >
 +                             virtual_memlimit_per_cpu_thread) {
-+                              /* If number of threads goes down to zero lzma_stream_encoder will
-+                               * will return UINT64_MAX. We must check here to avoid an infinite loop.
++                              /* If number of threads goes down to zero or in case of any other error
++                               * lzma_stream_encoder_mt_memusage will return UINT64_MAX. We must check
++                               * for both the cases here to avoid an infinite loop.
 +                               * If we get into situation that one thread requires more virtual memory
 +                               * than available we set one thread, print error message and try anyway. */
-+                              if (--mt_options.threads == 0) {
++                              if ((--mt_options.threads == 0) || (memory_usage_virt == UINT64_MAX)) {
 +                                      mt_options.threads = 1;
 +                                      rpmlog(RPMLOG_WARNING,
 +                                             "XZ: Could not adjust number of threads to get below "