meta/classes: Update classes to use new fetcher API

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index d8efcc0f8c4e96ab00bab9cc51766699607e1e40..edb65eb96b1c874dd1f170ad4bc41ea92eb3a17b 100644 (file)
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -116,163 +116,38 @@ addtask setscene before do_fetch
 addtask fetch
 do_fetch[dirs] = "${DL_DIR}"
 python base_do_fetch() {
-       import sys
+
+       src_uri = (bb.data.getVar('SRC_URI', d, True) or "").split()
+       if len(src_uri) == 0:
+               return
 
        localdata = bb.data.createCopy(d)
        bb.data.update_data(localdata)
 
-       src_uri = bb.data.getVar('SRC_URI', localdata, 1)
-       if not src_uri:
-               return 1
-
-       try:
-               bb.fetch.init(src_uri.split(),d)
-       except bb.fetch.NoMethodError:
-               (type, value, traceback) = sys.exc_info()
-               raise bb.build.FuncFailed("No method: %s" % value)
-       except bb.MalformedUrl:
-               (type, value, traceback) = sys.exc_info()
-               raise bb.build.FuncFailed("Malformed URL: %s" % value)
-
-       try:
-               if bb.fetch.__version__ == "1":
-                       bb.fetch.go(localdata)
-               else:
-                       bb.fetch.download(localdata)
-       except bb.fetch.MissingParameterError:
-               (type, value, traceback) = sys.exc_info()
-               raise bb.build.FuncFailed("Missing parameters: %s" % value)
-       except bb.fetch.FetchError:
-               (type, value, traceback) = sys.exc_info()
-               raise bb.build.FuncFailed("Fetch failed: %s" % value)
-       except bb.fetch.MD5SumError:
-               (type, value, traceback) = sys.exc_info()
-               raise bb.build.FuncFailed("MD5  failed: %s" % value)
-       except:
-               (type, value, traceback) = sys.exc_info()
-               raise bb.build.FuncFailed("Unknown fetch Error: %s" % value)
+        try:
+            fetcher = bb.fetch2.Fetch(src_uri, localdata)
+            fetcher.download()
+        except bb.fetch2.BBFetchException, e:
+            raise bb.build.FuncFailed(e)
 }
 
-def subprocess_setup():
-       import signal
-       # Python installs a SIGPIPE handler by default. This is usually not what
-       # non-Python subprocesses expect.
-       # SIGPIPE errors are known issues with gzip/bash
-       signal.signal(signal.SIGPIPE, signal.SIG_DFL)
-
-def oe_unpack_file(file, data, url = None):
-       import subprocess
-       if not url:
-               url = "file://%s" % file
-       dots = file.split(".")
-       if dots[-1] in ['gz', 'bz2', 'Z']:
-               efile = os.path.join(bb.data.getVar('WORKDIR', data, 1),os.path.basename('.'.join(dots[0:-1])))
-       else:
-               efile = file
-       cmd = None
-       if file.endswith('.tar'):
-               cmd = 'tar x --no-same-owner -f %s' % file
-       elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
-               cmd = 'tar xz --no-same-owner -f %s' % file
-       elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
-               cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file
-       elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
-               cmd = 'gzip -dc %s > %s' % (file, efile)
-       elif file.endswith('.bz2'):
-               cmd = 'bzip2 -dc %s > %s' % (file, efile)
-       elif file.endswith('.tar.xz'):
-               cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file
-       elif file.endswith('.xz'):
-               cmd = 'xz -dc %s > %s' % (file, efile)
-       elif file.endswith('.zip') or file.endswith('.jar'):
-               cmd = 'unzip -q -o'
-               (type, host, path, user, pswd, parm) = bb.decodeurl(url)
-               if 'dos' in parm:
-                       cmd = '%s -a' % cmd
-               cmd = "%s '%s'" % (cmd, file)
-       elif os.path.isdir(file):
-               filesdir = os.path.realpath(bb.data.getVar("FILESDIR", data, 1))
-               destdir = "."
-               if file[0:len(filesdir)] == filesdir:
-                       destdir = file[len(filesdir):file.rfind('/')]
-                       destdir = destdir.strip('/')
-                       if len(destdir) < 1:
-                               destdir = "."
-                       elif not os.access("%s/%s" % (os.getcwd(), destdir), os.F_OK):
-                               os.makedirs("%s/%s" % (os.getcwd(), destdir))
-               cmd = 'cp -pPR %s %s/%s/' % (file, os.getcwd(), destdir)
-       else:
-               (type, host, path, user, pswd, parm) = bb.decodeurl(url)
-               if not 'patch' in parm:
-                       # The "destdir" handling was specifically done for FILESPATH
-                       # items.  So, only do so for file:// entries.
-                       if type == "file" and path.find("/") != -1:
-                               destdir = path.rsplit("/", 1)[0]
-                       else:
-                               destdir = "."
-                       bb.mkdirhier("%s/%s" % (os.getcwd(), destdir))
-                       cmd = 'cp %s %s/%s/' % (file, os.getcwd(), destdir)
-
-       if not cmd:
-               return True
-
-       dest = os.path.join(os.getcwd(), os.path.basename(file))
-       if os.path.exists(dest):
-               if os.path.samefile(file, dest):
-                       return True
-
-       # Change to subdir before executing command
-       save_cwd = os.getcwd();
-       parm = bb.decodeurl(url)[5]
-       if 'subdir' in parm:
-               newdir = ("%s/%s" % (os.getcwd(), parm['subdir']))
-               bb.mkdirhier(newdir)
-               os.chdir(newdir)
-
-       cmd = "PATH=\"%s\" %s" % (bb.data.getVar('PATH', data, 1), cmd)
-       bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
-       ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
-
-       os.chdir(save_cwd)
-
-       return ret == 0
-
 addtask unpack after do_fetch
 do_unpack[dirs] = "${WORKDIR}"
 python base_do_unpack() {
-       import re
+       src_uri = (bb.data.getVar('SRC_URI', d, True) or "").split()
+       if len(src_uri) == 0:
+               return
 
        localdata = bb.data.createCopy(d)
        bb.data.update_data(localdata)
 
-       urldata = bb.fetch.init([], localdata)
+       rootdir = bb.data.getVar('WORKDIR', localdata, True)
 
-       src_uri = bb.data.getVar('SRC_URI', localdata, True)
-       if not src_uri:
-               return
-       for url in src_uri.split():
-               try:
-                       local = bb.data.expand(bb.fetch.localpath(url, localdata), localdata)
-               except bb.MalformedUrl, e:
-                       raise FuncFailed('Unable to generate local path for malformed uri: %s' % e)
-               if local is None:
-                       continue
-               local = os.path.realpath(local)
-               lockfile = urldata[url].lockfile
-               if lockfile:
-                       lf = bb.utils.lockfile(urldata[url].lockfile)
-
-               if bb.fetch.__version__ == "1":
-                       ret = oe_unpack_file(local, localdata, url)
-               else:
-                       # use bb.fetch2 unpack API
-                       ud = urldata[url]
-                       rootdir = bb.data.getVar('WORKDIR', localdata, True)
-                       ret = ud.method.unpack(ud, rootdir, localdata)
-               if lockfile:
-                       bb.utils.unlockfile(lf)
-               if not ret:
-                       raise bb.build.FuncFailed("oe_unpack_file failed with return value %s" % ret)
+        try:
+            fetcher = bb.fetch2.Fetch(src_uri, localdata)
+            fetcher.unpack(rootdir)
+        except bb.fetch2.BBFetchException, e:
+            raise bb.build.FuncFailed(e)
 }
 
 GIT_CONFIG = "${STAGING_DIR_NATIVE}/usr/etc/gitconfig"
@@ -550,7 +425,8 @@ python () {
             for s in srcuri.split():
                 if not s.startswith("file://"):
                     continue
-                local = bb.data.expand(bb.fetch.localpath(s, d), d)
+                fetcher = bb.fetch2.Fetch([s], d)
+                local = fetcher.localpath(s)
                 for mp in paths:
                     if local.startswith(mp):
                         #bb.note("overriding PACKAGE_ARCH from %s to %s" % (pkg_arch, mach_arch))
@@ -594,14 +470,12 @@ python do_cleanall() {
        dl_dir = bb.data.getVar('DL_DIR', localdata, True)
        dl_dir = os.path.realpath(dl_dir)
 
-       src_uri = bb.data.getVar('SRC_URI', localdata, True)
-       if not src_uri:
+       src_uri = (bb.data.getVar('SRC_URI', localdata, True) or "").split()
+       if len(src_uri) == 0:
                return
-       for url in src_uri.split():
-               try:
-                       local = bb.data.expand(bb.fetch.localpath(url, localdata), localdata)
-               except bb.MalformedUrl, e:
-                       raise FuncFailed('Unable to generate local path for malformed uri: %s' % e)
+        fetcher = bb.fetch2.Fetch(src_uri, localdata)
+       for url in src_uri:
+                local = fetcher.localpath(url)
                if local is None:
                        continue
                local = os.path.realpath(local)

diff --git a/meta/classes/patch.bbclass b/meta/classes/patch.bbclass
index ee8a2026fc8753faec9f5fae3fe61256ab8992be..80fd45f0e3c7b711ba99ae4084ff8b8989202e2a 100644 (file)
--- a/meta/classes/patch.bbclass
+++ b/meta/classes/patch.bbclass
@@ -58,9 +58,8 @@ python patch_do_patch() {
                        continue
 
                if not local:
-                       bb.fetch.init([url],d)
                        url = bb.encodeurl((type, host, path, user, pswd, []))
-                       local = os.path.join('/', bb.fetch.localpath(url, d))
+                       local = os.path.join('/', bb.fetch2.localpath(url, d))
                local = bb.data.expand(local, d)
 
                if "striplevel" in parm:

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index a754821c083e02e7bfdc493203473fed04898794..e4564e4b0716c828579b946b943a4142aad7454e 100644 (file)
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -326,40 +326,40 @@ def sstate_package(ss, d):
     return
 
 def pstaging_fetch(sstatepkg, d):
-    import bb.fetch
-
-    # only try and fetch if the user has configured a mirror
 
+    # Only try and fetch if the user has configured a mirror
     mirrors = bb.data.getVar('SSTATE_MIRRORS', d, True)
-    if mirrors:
-        # Copy the data object and override DL_DIR and SRC_URI
-        localdata = bb.data.createCopy(d)
-        bb.data.update_data(localdata)
+    if not mirrors:
+        return
 
-        dldir = bb.data.expand("${SSTATE_DIR}", localdata)
-        srcuri = "file://" + os.path.basename(sstatepkg)
+    import bb.fetch2
+    # Copy the data object and override DL_DIR and SRC_URI
+    localdata = bb.data.createCopy(d)
+    bb.data.update_data(localdata)
 
-        bb.mkdirhier(dldir)
+    dldir = bb.data.expand("${SSTATE_DIR}", localdata)
+    srcuri = "file://" + os.path.basename(sstatepkg)
 
-        bb.data.setVar('DL_DIR', dldir, localdata)
-        bb.data.setVar('PREMIRRORS', mirrors, localdata)
-        bb.data.setVar('SRC_URI', srcuri, localdata)
+    bb.mkdirhier(dldir)
 
-        # Try a fetch from the sstate mirror, if it fails just return and
-        # we will build the package
-        try:
-            bb.fetch.init([srcuri], localdata)
-            if bb.fetch.__version__ == "1":
-                bb.fetch.go(localdata, [srcuri])
-            else:
-                bb.fetch.download(localdata, [srcuri])
-            # Need to optimise this, if using file:// urls, the fetcher just changes the local path
-            # For now work around by symlinking
-            localpath = bb.data.expand(bb.fetch.localpath(srcuri, localdata), localdata)
-            if localpath != sstatepkg and os.path.exists(localpath):
-                os.symlink(localpath, sstatepkg)
-        except:
-            pass
+    bb.data.setVar('DL_DIR', dldir, localdata)
+    bb.data.setVar('PREMIRRORS', mirrors, localdata)
+    bb.data.setVar('SRC_URI', srcuri, localdata)
+
+    # Try a fetch from the sstate mirror, if it fails just return and
+    # we will build the package
+    try:
+        fetcher = bb.fetch2.Fetch([srcuri], localdata)
+        fetcher.download()        
+
+        # Need to optimise this, if using file:// urls, the fetcher just changes the local path
+        # For now work around by symlinking
+        localpath = bb.data.expand(fetcher.localpath(srcuri), localdata)
+        if localpath != sstatepkg and os.path.exists(localpath) and not os.path.exists(sstatepkg):
+            os.symlink(localpath, sstatepkg)
+
+    except bb.fetch2.BBFetchException:
+        pass
 
 def sstate_setscene(d):
     shared_state = sstate_state_fromvars(d)

diff --git a/meta/classes/utils.bbclass b/meta/classes/utils.bbclass
index 4d4b9953e23607fb1d3df723bb6be89fa8c67a9a..455b49d54ab1f2e5c5a5e8ce9d48c676e279a6ac 100644 (file)
--- a/meta/classes/utils.bbclass
+++ b/meta/classes/utils.bbclass
@@ -51,11 +51,12 @@ def machine_paths(d):
 def is_machine_specific(d):
     """Determine whether the current recipe is machine specific"""
     machinepaths = set(machine_paths(d))
-    urldatadict = bb.fetch.init(d.getVar("SRC_URI", True).split(), d, True)
-    for urldata in (urldata for urldata in urldatadict.itervalues()
-                    if urldata.type == "file"):
-        if any(urldata.localpath.startswith(mp + "/") for mp in machinepaths):
-            return True
+    srcuri = d.getVar("SRC_URI", True).split()
+    for url in srcuri:
+        fetcher = bb.fetch2.Fetch([srcuri], d)
+        if url.startswith("file://"):
+            if any(fetcher.localpath(url).startswith(mp + "/") for mp in machinepaths):
+                return True
 
 def oe_popen_env(d):
     env = d.getVar("__oe_popen_env", False)

diff --git a/meta/lib/oe/patch.py b/meta/lib/oe/patch.py
index f203d683da22ff18d4cb0156fe1427f050a3a25e..c8eeb8bc4cca55dce993bc0801f9812b1799dd61 100644 (file)
--- a/meta/lib/oe/patch.py
+++ b/meta/lib/oe/patch.py
@@ -72,14 +72,14 @@ class PatchSet(object):
             if not patch.get("remote"):
                 raise PatchError("Patch file must be specified in patch import.")
             else:
-                patch["file"] = bb.fetch.localpath(patch["remote"], self.d)
+                patch["file"] = bb.fetch2.localpath(patch["remote"], self.d)
 
         for param in PatchSet.defaults:
             if not patch.get(param):
                 patch[param] = PatchSet.defaults[param]
 
         if patch.get("remote"):
-            patch["file"] = bb.data.expand(bb.fetch.localpath(patch["remote"], self.d), self.d)
+            patch["file"] = bb.data.expand(bb.fetch2.localpath(patch["remote"], self.d), self.d)
 
         patch["filemd5"] = bb.utils.md5_file(patch["file"])
 
@@ -293,7 +293,7 @@ class QuiltTree(PatchSet):
             if type == "file":
                 import shutil
                 if not patch.get("file") and patch.get("remote"):
-                    patch["file"] = bb.fetch.localpath(patch["remote"], self.d)
+                    patch["file"] = bb.fetch2.localpath(patch["remote"], self.d)
 
                 shutil.copyfile(patch["quiltfile"], patch["file"])
             else: