bitbake: unify mirror support and make it independant of the fetcher

This patch serves two purposes. Firstly it unifies the concept of mirrors into
PREMIRRORS and MIRRORS. PREMIRRORS are tried before the SRC_URI defined in the
recipe whereas MIRRORS are tried only if that fails.
The tarball stash was conceptually inline with a PREMIRROR only with special
handling within the wget fetcher and therefore only worked with certain
fetch types.
Secondly the patch removes the need for individual fetch implementations to
worry about mirror handling.

With this patch, the base fetch implementation will first try to use a
PREMIRROR to fetch the desired object, if this fails the native fetch method
for the object will be tried and if this fails will try to fetch a copy from
one of the MIRRORS.

Signed-off-by: Joshua Lock <josh@linux.intel.com>

diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py
index ab4658bc3b668526dd37b258f864382c587dc25a..ccb60de59c9d46bbe8aae18081a913ba907eb08a 100644 (file)
--- a/bitbake/lib/bb/fetch/__init__.py
+++ b/bitbake/lib/bb/fetch/__init__.py
@@ -185,7 +185,18 @@ def go(d, urls = None):
                     pass
                 bb.utils.unlockfile(lf)
                 continue
-        m.go(u, ud, d)
+
+        # First try fetching uri, u, from PREMIRRORS
+        mirrors = [ i.split() for i in (bb.data.getVar('PREMIRRORS', d, 1) or "").split('\n') if i ]
+        if not  try_mirrors(d, u, mirrors):
+            # Next try fetching from the original uri, u
+            try:
+                m.go(u, ud, d)
+            except:
+                # Finally, try fetching uri, u, from MIRRORS
+                mirrors = [ i.split() for i in (bb.data.getVar('MIRRORS', d, 1) or "").split('\n') if i ]
+                try_mirrors (d, u, mirrors)
+
         if ud.localfile:
             if not m.forcefetch(u, ud, d):
                 Fetch.write_md5sum(u, ud, d)
@@ -332,6 +343,44 @@ def runfetchcmd(cmd, d, quiet = False):
 
     return output
 
+def try_mirrors(d, uri, mirrors):
+    """
+    Try to use a mirrored version of the sources.
+    This method will be automatically called before the fetchers go.
+
+    d Is a bb.data instance
+    uri is the original uri we're trying to download
+    mirrors is the list of mirrors we're going to try
+    """
+    fpath = os.path.join(data.getVar("DL_DIR", d, 1), os.path.basename(uri))
+    if os.access(fpath, os.R_OK):
+        bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % fpath)
+        return True
+
+    ld = d.createCopy()
+    for (find, replace) in mirrors:
+        newuri = uri_replace(uri, find, replace, ld)
+        if newuri != uri:
+            try:
+                ud = FetchData(newuri, ld)
+            except bb.fetch.NoMethodError:
+                bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url)
+                continue
+
+            ud.setup_localpath(ld)
+
+            try:
+                ud.method.go(newuri, ud, ld)
+                return True
+            except (bb.fetch.MissingParameterError,
+                    bb.fetch.FetchError,
+                    bb.fetch.MD5SumError):
+                import sys
+                (type, value, traceback) = sys.exc_info()
+                bb.msg.debug(2, bb.msg.domain.Fetcher, "Mirror fetch failure: %s" % value)
+                return False
+
+
 class FetchData(object):
     """
     A class which represents the fetcher state for a given URI.
@@ -489,49 +538,6 @@ class Fetch(object):
 
     localcount_internal_helper = staticmethod(localcount_internal_helper)
 
-    def try_mirror(d, tarfn):
-        """
-        Try to use a mirrored version of the sources. We do this
-        to avoid massive loads on foreign cvs and svn servers.
-        This method will be used by the different fetcher
-        implementations.
-
-        d Is a bb.data instance
-        tarfn is the name of the tarball
-        """
-        tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn)
-        if os.access(tarpath, os.R_OK):
-            bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn)
-            return True
-
-        pn = data.getVar('PN', d, True)
-        src_tarball_stash = None
-        if pn:
-            src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split()
-
-        ld = d.createCopy()
-        for stash in src_tarball_stash:
-            url = stash + tarfn
-            try:
-                ud = FetchData(url, ld)
-            except bb.fetch.NoMethodError:
-                bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url)
-                continue
-
-            ud.setup_localpath(ld)
-
-            try:
-                ud.method.go(url, ud, ld)
-                return True
-            except (bb.fetch.MissingParameterError,
-                    bb.fetch.FetchError,
-                    bb.fetch.MD5SumError):
-                import sys
-                (type, value, traceback) = sys.exc_info()
-                bb.msg.debug(2, bb.msg.domain.Fetcher, "Tarball stash fetch failure: %s" % value)
-        return False
-    try_mirror = staticmethod(try_mirror)
-
     def verify_md5sum(ud, got_sum):
         """
         Verify the md5sum we wanted with the one we got

diff --git a/bitbake/lib/bb/fetch/bzr.py b/bitbake/lib/bb/fetch/bzr.py
index b27fb63d072c42a88c53dc676bc87af7daba5ef5..c6e33c3343b4c0d1ede19fe1d58bfd0767637872 100644 (file)
--- a/bitbake/lib/bb/fetch/bzr.py
+++ b/bitbake/lib/bb/fetch/bzr.py
@@ -91,11 +91,6 @@ class Bzr(Fetch):
     def go(self, loc, ud, d):
         """Fetch url"""
 
-        # try to use the tarball stash
-        if Fetch.try_mirror(d, ud.localfile):
-            bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping bzr checkout." % ud.localpath)
-            return
-
         if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
             bzrcmd = self._buildbzrcommand(ud, d, "update")
             bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Update %s" % loc)

diff --git a/bitbake/lib/bb/fetch/cvs.py b/bitbake/lib/bb/fetch/cvs.py
index 90a006500e870c3c2e8e8dd0076f0ff69ebb246b..443f52131786039c11afd76ab93a733b357c5a3a 100644 (file)
--- a/bitbake/lib/bb/fetch/cvs.py
+++ b/bitbake/lib/bb/fetch/cvs.py
@@ -77,11 +77,6 @@ class Cvs(Fetch):
 
     def go(self, loc, ud, d):
 
-        # try to use the tarball stash
-        if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
-            bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping cvs checkout." % ud.localpath)
-            return
-
         method = "pserver"
         if "method" in ud.parm:
             method = ud.parm["method"]

diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py
index 9430582d552aa1e710d1baae397101d7a2d14e90..10b396bb00415c809be3d3606652b05c983b2087 100644 (file)
--- a/bitbake/lib/bb/fetch/git.py
+++ b/bitbake/lib/bb/fetch/git.py
@@ -82,10 +82,6 @@ class Git(Fetch):
     def go(self, loc, ud, d):
         """Fetch url"""
 
-        if Fetch.try_mirror(d, ud.localfile):
-            bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
-            return
-
         if ud.user:
             username = ud.user + '@'
         else:
@@ -97,11 +93,12 @@ class Git(Fetch):
         codir = os.path.join(ud.clonedir, coname)
 
         if not os.path.exists(ud.clonedir):
-            if Fetch.try_mirror(d, ud.mirrortarball):    
+            try:
+                Fetch.try_mirrors(ud.mirrortarball)
                 bb.mkdirhier(ud.clonedir)
                 os.chdir(ud.clonedir)
                 runfetchcmd("tar -xzf %s" % (repofile), d)
-            else:
+            except:
                 runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.clonedir), d)
 
         os.chdir(ud.clonedir)

diff --git a/bitbake/lib/bb/fetch/hg.py b/bitbake/lib/bb/fetch/hg.py
index 08cb61fc286e28289bc6498a95d88e362c95e5b1..d0756382f88ef8ea83f22d1b3d26bb37f540c23c 100644 (file)
--- a/bitbake/lib/bb/fetch/hg.py
+++ b/bitbake/lib/bb/fetch/hg.py
@@ -116,11 +116,6 @@ class Hg(Fetch):
     def go(self, loc, ud, d):
         """Fetch url"""
 
-        # try to use the tarball stash
-        if Fetch.try_mirror(d, ud.localfile):
-            bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping hg checkout." % ud.localpath)
-            return
-
         bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
 
         if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):

diff --git a/bitbake/lib/bb/fetch/osc.py b/bitbake/lib/bb/fetch/osc.py
index 2c34caf6c9a61651d2ad5a414ac63f12e99e167f..548dd9d0746f5de7adc52fc9114bdc14d9f8be3e 100644 (file)
--- a/bitbake/lib/bb/fetch/osc.py
+++ b/bitbake/lib/bb/fetch/osc.py
@@ -91,11 +91,6 @@ class Osc(Fetch):
         Fetch url
         """
 
-        # Try to use the tarball stash
-        if Fetch.try_mirror(d, ud.localfile):
-            bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping osc checkout." % ud.localpath)
-            return
-
         bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
 
         if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK):

diff --git a/bitbake/lib/bb/fetch/perforce.py b/bitbake/lib/bb/fetch/perforce.py
index 394f5a225331dcea7d91ebbe9daf27c32c679859..e2c3421089014965a6d5d3ae10f617fe7286cd8f 100644 (file)
--- a/bitbake/lib/bb/fetch/perforce.py
+++ b/bitbake/lib/bb/fetch/perforce.py
@@ -124,11 +124,6 @@ class Perforce(Fetch):
         Fetch urls
         """
 
-        # try to use the tarball stash
-        if Fetch.try_mirror(d, ud.localfile):
-            bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping perforce checkout." % ud.localpath)
-            return
-
         (host,depot,user,pswd,parm) = Perforce.doparse(loc, d)
 
         if depot.find('/...') != -1:

diff --git a/bitbake/lib/bb/fetch/svk.py b/bitbake/lib/bb/fetch/svk.py
index 120dad9d4e9f53680e8d3893944a4d532276fd50..a17ac04d2108db33d8c2bbfa5019efb6ec0df956 100644 (file)
--- a/bitbake/lib/bb/fetch/svk.py
+++ b/bitbake/lib/bb/fetch/svk.py
@@ -62,9 +62,6 @@ class Svk(Fetch):
     def go(self, loc, ud, d):
         """Fetch urls"""
 
-        if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
-            return
-
         svkroot = ud.host + ud.path
 
         svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module)

diff --git a/bitbake/lib/bb/fetch/svn.py b/bitbake/lib/bb/fetch/svn.py
index eef9862a84abc27d3d93dc1d1b08e006db0d30f0..ba9f6ab1098b20c67e8dfa76adf459c1c8e2b33d 100644 (file)
--- a/bitbake/lib/bb/fetch/svn.py
+++ b/bitbake/lib/bb/fetch/svn.py
@@ -136,11 +136,6 @@ class Svn(Fetch):
     def go(self, loc, ud, d):
         """Fetch url"""
 
-        # try to use the tarball stash
-        if Fetch.try_mirror(d, ud.localfile):
-            bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping svn checkout." % ud.localpath)
-            return
-
         bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
 
         if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):

diff --git a/bitbake/lib/bb/fetch/wget.py b/bitbake/lib/bb/fetch/wget.py
index fd93c7ec46a1ca9002b60abf65fd7cd2681fca2b..ae1c6ad13632a3dfd6475a290769f621117dcc5f 100644 (file)
--- a/bitbake/lib/bb/fetch/wget.py
+++ b/bitbake/lib/bb/fetch/wget.py
@@ -30,7 +30,6 @@ import bb
 from   bb import data
 from   bb.fetch import Fetch
 from   bb.fetch import FetchError
-from   bb.fetch import uri_replace
 
 class Wget(Fetch):
     """Class to fetch urls via 'wget'"""
@@ -105,24 +104,9 @@ class Wget(Fetch):
         data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata)
         data.update_data(localdata)
 
-        premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ]
-        for (find, replace) in premirrors:
-            newuri = uri_replace(uri, find, replace, d)
-            if newuri != uri:
-                if fetch_uri(newuri, ud, localdata):
-                    return True
-
         if fetch_uri(uri, ud, localdata):
             return True
 
-        # try mirrors
-        mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ]
-        for (find, replace) in mirrors:
-            newuri = uri_replace(uri, find, replace, d)
-            if newuri != uri:
-                if fetch_uri(newuri, ud, localdata):
-                    return True
-
         raise FetchError(uri)